{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先 import 必要的模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from xgboost import XGBClassifier\n",
    "import lightgbm as lgbm\n",
    "#from lgbm.sklearn import LGBMClassifier\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import log_loss     #采用logloss作为评价指标\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7932959, 23)"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(\"D:/Jupyter/w000/train_final/800w/800_0\",index_col=0)   #拿200万条数据训练\n",
    "train.drop(['id','hour','user_id','Unnamed: 0'],axis=1,inplace=True)\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3936502, 23)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid = pd.read_csv(\"D:/Jupyter/w000/train_final/400w/400_1\",index_col=0)   #拿200万条数据训练\n",
    "valid.drop(['id','hour','user_id','Unnamed: 0'],axis=1,inplace=True)\n",
    "valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\numpy\\lib\\arraysetops.py:466: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  mask |= (ar1 == a)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>click</th>\n",
       "      <th>C1</th>\n",
       "      <th>banner_pos</th>\n",
       "      <th>site_id</th>\n",
       "      <th>site_domain</th>\n",
       "      <th>site_category</th>\n",
       "      <th>app_id</th>\n",
       "      <th>app_domain</th>\n",
       "      <th>app_category</th>\n",
       "      <th>device_model</th>\n",
       "      <th>...</th>\n",
       "      <th>C15</th>\n",
       "      <th>C16</th>\n",
       "      <th>C17</th>\n",
       "      <th>C18</th>\n",
       "      <th>C19</th>\n",
       "      <th>C20</th>\n",
       "      <th>C21</th>\n",
       "      <th>hour_hours</th>\n",
       "      <th>hour_weekday</th>\n",
       "      <th>hour_days</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>36210029</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>86</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>140</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>181</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>53</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36210030</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>86</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>181</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36210031</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>128</td>\n",
       "      <td>109</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>53</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36210032</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>98</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36210033</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>59</td>\n",
       "      <td>58</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>111</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          click  C1  banner_pos  site_id  site_domain  site_category  app_id  \\\n",
       "36210029      0   0           1       86            2              1       0   \n",
       "36210030      0   0           1       86            2              1       0   \n",
       "36210031      0   0           0      128          109              3       0   \n",
       "36210032      0   0           1        5            5              1       0   \n",
       "36210033      0   0           0       59           58              1       0   \n",
       "\n",
       "          app_domain  app_category  device_model    ...      C15  C16  C17  \\\n",
       "36210029           0             0           140    ...        0    0  181   \n",
       "36210030           0             0             7    ...        0    0  181   \n",
       "36210031           0             0            53    ...        0    0   27   \n",
       "36210032           0             0            98    ...        0    0    2   \n",
       "36210033           0             0           111    ...        0    0   35   \n",
       "\n",
       "          C18  C19  C20  C21  hour_hours  hour_weekday  hour_days  \n",
       "36210029    0    0   53   11           0             4          9  \n",
       "36210030    0    0   34   11           0             4          9  \n",
       "36210031    1    1    2    0           0             4          9  \n",
       "36210032    0   10    0    3           0             4          9  \n",
       "36210033    1    4    0    0           0             4          9  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv(\"D:/Jupyter/w000/train_final/FE_test_26_final.csv\",index_col=0)   #拿200万条数据训练\n",
    "test.drop(['id','hour','user_id'],axis=1,inplace=True)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variable Identification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "选择该数据集是因为的数据特征单一，我们可以在特征工程方面少做些工作，集中精力放在参数调优上"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['click']\n",
    "X_train = train.drop([\"click\"], axis=1)\n",
    "\n",
    "y_valid = valid['click']\n",
    "X_valid = valid.drop([\"click\"], axis=1)\n",
    "\n",
    "y_test = test['click']\n",
    "X_test = test.drop([\"click\"], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "默认参数，此时学习率为0.1，比较大，观察弱分类数目的大致范围\n",
    "（采用默认参数配置，看看模型是过拟合还是欠拟合）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#直接调用LightGBM内嵌的交叉验证（cv），可对连续的n_estimators参数进行快速交叉验证\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证\n",
    "import json\n",
    "def modelfit(params, alg, X_train, y_train,early_stopping_rounds=50):\n",
    "    lgbm_params = params.copy()\n",
    "    \n",
    "    lgbmtrain = lgbm.Dataset(X_train, y_train, silent=True,categorical_feature=[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21])\n",
    "\n",
    "    \n",
    "    cv_result = lgbm.cv(\n",
    "        lgbm_params, lgbmtrain, num_boost_round=10000, nfold=3, stratified=False, shuffle=True, metrics='binary_logloss',\n",
    "        early_stopping_rounds=early_stopping_rounds,show_stdv=True,seed=0)\n",
    "    # note: cv_results will look like: {\"multi_logloss-mean\": <a list of historical mean>,\n",
    "    # \"multi_logloss-stdv\": <a list of historical standard deviation>}\n",
    "    print('best n_estimators:', len(cv_result['binary_logloss-mean']))\n",
    "    print('best cv logloss:', cv_result['binary_logloss-mean'][-1])\n",
    "    #cv_result.to_csv('lgbm1_nestimators.csv', index_label = 'n_estimators')\n",
    "    json.dump(cv_result, open('lgbm_1.json', 'w'))\n",
    "    #\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = len(cv_result['binary_logloss-mean']))\n",
    "    alg.fit(X_train, y_train)\n",
    "\n",
    "    y_pred_train = alg.predict_proba(X_train)\n",
    "    logloss = log_loss(y_train, y_pred_train)\n",
    "    print (\"logloss of train :\",logloss )\n",
    "    \n",
    "    y_pred_test = alg.predict_proba(X_test)\n",
    "    logloss = log_loss(y_test, y_pred_test)\n",
    "    print (\"logloss of test :\",logloss )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "params = {'boosting_type': 'gbdt', \n",
    "          'objective': 'binary', \n",
    "          'nthread': 6, \n",
    "          #default=OpenMP_default, type=int, alias=num_thread, nthread LightGBM 的线程数\n",
    "          'silent': True,\n",
    "          'learning_rate': 0.1, \n",
    "          'num_leaves': 50, \n",
    "          #num_leaves：叶子数\n",
    "#num_leaves, default=31, type=int, alias=num_leaf一棵树上的叶子数\n",
    "          'max_depth': 6,\n",
    "          'max_bin': 256, \n",
    "          'subsample_for_bin': 200000,#默认200000，用来构建直方图数据数量\n",
    "          'subsample': 0.8, #默认1\n",
    "          #类似于feature_fraction,它将在不进行重采样的情况下\n",
    "          #随机选择部分数据可以用来加速训练可以用来处理过拟合Not\n",
    "          'subsample_freq': 1, \n",
    "          'colsample_bytree': 0.8,       \n",
    "          'reg_alpha': 0,         \n",
    "          'reg_lambda': 0, \n",
    "          'min_split_gain': 0.0, \n",
    "          'min_child_weight': 1,        \n",
    "          'min_child_samples': 100, \n",
    "          'min_data_in_leaf':2000, \n",
    "          #default=20, type=int, alias=min_data_per_leaf , min_data, \n",
    "          #min_child_samples一个叶子上数据的最小数量. 可以用来处理过拟合.\n",
    "        \n",
    "\n",
    "         }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1040: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n",
      "C:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:661: UserWarning: silent keyword has been found in `params` and will be ignored. Please use silent argument of the Dataset constructor to pass this parameter.\n",
      "  'Please use {0} argument of the Dataset constructor to pass this parameter.'.format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best n_estimators: 363\n",
      "best cv logloss: 0.3998200300886134\n",
      "logloss of train : 0.40013086259651365\n",
      "logloss of test : 0.4039739125311909\n",
      "time cost255.63825130462646s\n"
     ]
    }
   ],
   "source": [
    "time_start=time.time()\n",
    "\n",
    "lgbm1 = lgbm.sklearn.LGBMClassifier(n_estimators=1000,seed=0, **params)\n",
    "modelfit(params,lgbm1, X_train, y_train)\n",
    "\n",
    "time_end=time.time()\n",
    "a=time_end - time_start\n",
    "print('time cost'+str(a)+'s')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEXCAYAAABcRGizAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XucXWV97/HPd++5X3KfQO4JISgRLWhApBU5XiraCmgRQVGxF8TKi9pCK9hqLdTWcnpEPVAVW6BSFVHqMSKKHkvq5UjIoIAEBEIuEBLI/TrJTGbmd/5Yz0x2Jntm7ySzs4eZ7/v12q9Z61nPWuuXlZn5zfM8a61HEYGZmdlQctUOwMzMRj4nCzMzK8nJwszMSnKyMDOzkpwszMysJCcLMzMrycnCjjpJqyW9sUj5ayU9UY2YBiPpNkl/PwzHuUTSz4YjJrNqqKl2AGZ9IuKnwEuqHYeZHcwtCxv1lPH3utkR8A+QVcupkh6TtFXSrZIaJJ0laW1fhdRddZWkRyRtl/QNSQ1p20RJd0vamI5xt6SZBfsukfQpST8HOoArJT1YGICkKyX9n0MJWtKfSFohaYukxZKmF2z7XUlPpFj/RdJ/S/rjQY5zhqRlqe4ySWcUbLtE0kpJOyWtkvSeVH58OuZ2SZskfWOIOG9LMXxf0i5JP5d0rKTPpuv1G0mnFNSfLumudD1XSbqiYNtpkn4haZuk9ZJulFRXsD0kXSbpqXTsmyTpUK6rjXxOFlYt7wHeDMwHTgD+ZpB6FwBnA/OAVwCXpPIccCswB5gN7AFuHLDve4FLgVbg88A8SScWbL8YuL3cgCW9HvjHFNM0YA1wR9o2BfgWcA0wGXgCOGOQ40wCvpdimgx8BviepMmSmlP5WyKiNR3jobTrdcAPgYnATOB/lwj5ArLrOgXoBH4B/DKtfyudl9Tq+i7wMDADeAPwEUlvTsfpAf487featP1PB5zr94FTgd9K530zNqo4WVi13BgRz0bEFuBTwEWD1Pt8RKxL9b4LnAwQEZsj4q6I6IiInekYrxuw720RsTwiuiOiE/gGWYJA0suAucDdhxDze4BbIuKX6XjXAK+RNBd4K7A8Iv4zIrrJfuE/P8hxfg94KiJuT7F9HfgN8La0vRc4SVJjRKyPiOWpfB9ZcpweEXsjotSA+bcj4sGI2At8G9gbEV+JiJ50LfpaFqcCbRFxbUR0RcRK4MvAhQDpGPenWFcDX+Lga/3piNgWEc8A95H+n2z0cLKwanm2YHkNMH2QeoW/cDuAFgBJTZK+JGmNpB3AT4AJkvKDnAPg34F3py6S9wJ3pl/65ZqeYgUgInYBm8n+Gp9eeL7I3tC5duABih0nWQPMiIjdwLuAy4D1kr4n6aWpzl8BAh6QtFzSHwJI+ljqatol6YsFx3yhYHlPkfWWtDwHmJ66mbZJ2gZ8DDgmHf+E1M33fLrW/0DWyihU9P/JRg8nC6uWWQXLs4F1h7j/lWR3Tr06IsYBZ6bywr7yA16pHBH3A13Aa4F3cwhdUMk6sl+s2YmyLqPJwHPAerKuob5tKlwf6jjJ7HQcIuLeiHgTWVfXb8j+yicino+IP4mI6cAHgX+RdHxE/ENEtKTPZYf4b4Isya2KiAkFn9aIeGva/oUUx4J0rT/GgdfZxgAnC6uWD0uamfrvP0bWLXIoWsn+Ot6WjvG3Ze73FbKxje4yunEG+hrwAUknS6on+wt7aeqa+R7wcknnSaoBPgwcO8hx7gFOkPRuSTWS3gUsBO6WdIykc1Ii6gR2kY0ZIOmdBYP4W8mSYc8h/huKeQDYIemjkhol5SWdJOnUtL0V2AHsSq2cDw3DOe1FxsnCquVrZIO1K9PnUB98+yzQCGwC7gd+UOZ+twMnceitCiLix8DHgbvIWhLz2d+vvwl4J3A9WdfUQqCd7Bf+wONsJhsQvjLV/Svg99Mxcql8HbCFbGygbzD5VGCppF3AYuDPImLVof47isTTQzZecjKwiuya/iswPlW5iqwltpOslXOoid1GAXnyIxtLJDUCG4BXRsRTFTxPjmzM4j0RcV+lzmN2tLhlYWPNh4BllUgUkt4saULqourr179/uM9jVg1+3YeNGZJWk/0CP29A+XIOHnAG+GBEfPUQTvEasu61OuAx4LyI2HN40ZqNLO6GMjOzktwNZWZmJY2abqgpU6bE3Llzqx2GmdmLyoMPPrgpItpK1Rs1yWLu3Lm0t7dXOwwzsxcVSQPfJlCUu6HMzKwkJwszMyvJycLMzEpysjAzs5IqmiwknZ1mDlsh6eoh6p2fZttalNbnStoj6aH0+eJg+5qZWeVV7G6oNK/ATcCbyN6Rs0zS4oh4bEC9VuAKYOmAQzwdEZ5AxcxsBKhky+I0YEVErIyILrLpJ88tUu86sjd17q1gLGZmdgQqmSxmcOBMZWtTWb80YfysiCg2teU8Sb9KE9S/ttgJJF0qqV1S+8aNGw8ryI07O7n9F6tZu7XjsPY3MxsLKpksis2k1f8iqvQK5xvI3t0/0HpgdkScAvwF8DVJ4w46WMTNEbEoIha1tZV8ALGo9dv38PHvLOeJ53ce1v5mZmNBJZPFWg6cOnMmB06d2Uo2Cc2S9DbQ04HFkhZFRGeaIIaIeBB4GjihEkHW12RTNnd291bi8GZmo0Ilk8UyYIGkeZLqyGYUW9y3MSK2R8SUiJgbEXPJ3vt/TkS0S2pLA+RIOg5YQDab2rCrr8kuQWf3cMxOaWY2OlXsbqiI6JZ0OXAvkAduiYjlkq4F2iNi8RC7nwlcK6mbbI7hyyJiSyXirK9NyWKfWxZmZoOp6IsEI+IessnpC8s+MUjdswqW7yKb57ji3A1lZlbamH+C291QZmaljflkUVfjbigzs1LGfLKoyYmc3A1lZjaUMZ8sJFFfk3c3lJnZEMZ8soDsjii3LMzMBudkQTbI7TELM7PBOVmAu6HMzEpwsiC1LNwNZWY2KCcLPGZhZlaKkwXuhjIzK8XJAg9wm5mV4mSBxyzMzEpxssDdUGZmpThZ4AFuM7NSnCzwmIWZWSlOFrgbysysFCcLPMBtZlaKkwUeszAzK8XJgqwbqqc36O5xwjAzK8bJgsKpVZ0szMyKcbLAycLMrBQnC6C+Ng/gO6LMzAbhZAE01GaXYa+ftTAzK8rJAmhMLYu9+9yyMDMrxskCaEjJYo+ThZlZUU4WFLQsupwszMyKcbIAGuvcsjAzG4qTBftbFk4WZmbFOVlQMGbhbigzs6KcLNjfDeW7oczMinOywN1QZmalVDRZSDpb0hOSVki6eoh650sKSYsGlM+WtEvSVZWMc383lB/KMzMrpmLJQlIeuAl4C7AQuEjSwiL1WoErgKVFDnMD8P1KxdgnnxN1NTm3LMzMBlHJlsVpwIqIWBkRXcAdwLlF6l0HXA/sLSyUdB6wElhewRj7NdbmPWZhZjaISiaLGcCzBetrU1k/SacAsyLi7gHlzcBHgb8b6gSSLpXULql948aNRxRsY23ed0OZmQ2ikslCRcqif6OUI+tmurJIvb8DboiIXUOdICJujohFEbGora3tiIJtrMu7G8rMbBA1FTz2WmBWwfpMYF3BeitwErBEEsCxwGJJ5wCvBs6XdD0wAeiVtDcibqxUsA21ThZmZoOpZLJYBiyQNA94DrgQeHffxojYDkzpW5e0BLgqItqB1xaUfxLYVclEAdBYm/OYhZnZICrWDRUR3cDlwL3A48CdEbFc0rWp9TCiNNZ5zMLMbDCVbFkQEfcA9wwo+8Qgdc8apPyTwx5YEY21ebZ17DsapzIze9HxE9yJxyzMzAbnZJE01uY9n4WZ2SCcLBLfOmtmNjgni6SxNk+HWxZmZkU5WSSNdXk6u3vp6Y3Slc3Mxhgni6S5LrsxzF1RZmYHc7JImuqz15R3dHZXORIzs5HHySLpa1ns9riFmdlBnCySvqlVd7tlYWZ2ECeLxGMWZmaDc7JI+sYs3LIwMzuYk0XS17LwsxZmZgdzskiaPGZhZjYoJ4ukL1m4ZWFmdjAni6S5vu/WWbcszMwGcrJI6mty5AQdnW5ZmJkN5GSRSKK5rsbdUGZmRThZFGisy9Phbigzs4M4WRRorq/x6z7MzIpwsijQVJf3iwTNzIpwsijQXFfju6HMzIpwsijQVO/Z8szMinGyKNBcV8Mud0OZmR3EyaJAS30Nu/Y6WZiZDeRkUaC1wS0LM7NinCwKtDRkD+X19Ea1QzEzG1GcLAq0pPdDuXVhZnYgJ4sCrQ1OFmZmxThZFGiprwXwILeZ2QBOFgVa+lsW+6ociZnZyFLRZCHpbElPSFoh6eoh6p0vKSQtSuunSXoofR6W9PZKxtmnb8xip1sWZmYHqKnUgSXlgZuANwFrgWWSFkfEYwPqtQJXAEsLih8FFkVEt6RpwMOSvhsRFf0t7jELM7PiKtmyOA1YERErI6ILuAM4t0i964Drgb19BRHRUZAYGoCjci9r/91QblmYmR2gksliBvBswfraVNZP0inArIi4e+DOkl4taTnwa+CyYq0KSZdKapfUvnHjxiMOuMUtCzOzokomC0nzJdWn5bMkXSFpQhnHVpGy/haCpBxwA3BlsZ0jYmlEvAw4FbhGUkOROjdHxKKIWNTW1lZGSENrrvOYhZlZMeW0LO4CeiQdD/wbMA/4Whn7rQVmFazPBNYVrLcCJwFLJK0GTgcW9w1y94mIx4HdqW5F5XOiuS7vZGFmNkA5yaI3dQG9HfhsRPw5MK2M/ZYBCyTNk1QHXAgs7tsYEdsjYkpEzI2IucD9wDkR0Z72qQGQNAd4CbD6UP5hh6uloca3zpqZDVDO3VD7JF0EvB94WyqrLbVTupPpcuBeIA/cEhHLJV0LtEfE4iF2/x3gakn7gF7gTyNiUxmxHrHWhlp27HHLwsysUDnJ4gPAZcCnImKVpHnAf5Rz8Ii4B7hnQNknBql7VsHy7cDt5ZxjuE1sqmXbnq5qnNrMbMQqmSzScxFXAEiaCLRGxKcrHVi1TGiq49ktHdUOw8xsRCnnbqglksZJmgQ8DNwq6TOVD606JjbVsrXDLQszs0LlDHCPj4gdwDuAWyPiVcAbKxtW9UxsqmNrxz4iPKeFmVmfcpJFTXrlxgXAQQ/PjTYTmuro6u5lz76eaodiZjZilJMsriW7o+npiFgm6TjgqcqGVT0Tm7IbvbZ2+PZZM7M+5QxwfxP4ZsH6SuAPKhlUNU1oqgNg6+4uZkxorHI0ZmYjQzkD3DMlfVvSBkkvSLpL0syjEVw19LUstrllYWbWr5xuqFvJnryeTvYiwO+mslFpYnNqWfiOKDOzfuUki7aIuDUiutPnNuDI39o3Qk3oH7NwsjAz61NOstgk6WJJ+fS5GNhc6cCqZUJj35iFu6HMzPqUkyz+kOy22eeB9cD5ZK8AGZXqanK01Ne4ZWFmVqBksoiIZyLinIhoi4ipEXEe2QN6o9aEplq2OVmYmfU73Jny/mJYoxhh+p7iNjOzzOEmi2Kz4I0ablmYmR3ocJPFqH5xklsWZmYHGvQJbkk7KZ4UBIzqR5v95lkzswMNmiwiovVoBjKSTGiqY+febrp7eqnJH27jy8xs9PBvwiImpae4t+1xV5SZGThZFDWh//1Q7ooyMwMni6Im9r151oPcZmaAk0VREwteU25mZmXMZzHIXVHbgXbgyjS/xagywa8pNzM7QMlkAXwGWAd8jey22QuBY4EngFuAsyoVXLX4NeVmZgcqpxvq7Ij4UkTsjIgdEXEz8NaI+AYwscLxVUVzXZ7avDxmYWaWlJMseiVdICmXPhcUbBuVT3JLYkJTne+GMjNLykkW7wHeC2xIn/cCF0tqBC6vYGxV5ae4zcz2KzlmkQaw3zbI5p8NbzgjxwS/H8rMrF/JloWkmZK+LWmDpBck3SVp5tEIrpom+s2zZmb9yumGuhVYDEwHZgDfTWWjmt88a2a2XznJoi0ibo2I7vS5DWircFxV1zfAHTEqx/DNzA5JOclik6SLJeXT52Jgc6UDq7aJTbXs6wl2d/VUOxQzs6orJ1n8IXAB8DywHjgf+EA5B5d0tqQnJK2QdPUQ9c6XFJIWpfU3SXpQ0q/T19eXc77h5Fd+mJntVzJZRMQzEXFORLRFxNSIOA94R6n9JOWBm4C3AAuBiyQtLFKvFbgCWFpQvAl4W0S8HHg/cHtZ/5phNLklSxabdnUe7VObmY04h/siwb8oo85pwIqIWBkRXcAdwLlF6l0HXA/s7SuIiF9FxLq0uhxokFR/mLEelmPHNwDwwo69JWqamY1+h5ssVEadGcCzBetrU9n+g0inALMi4u4hjvMHwK8i4qA/8SVdKqldUvvGjRvLCKl808ZnM8eu2+ZkYWZ2uMminFuEiiWU/v0k5YAbgCsHPYD0MuCfgA8WDSLi5ohYFBGL2tqG9watiU211NfkeN4tCzOzwZ/gHuTV5JAlgcYyjr0WmFWwPpPs7bV9WoGTgCWSIHuT7WJJ50REe3rw79vA+yLi6TLON6wkMW18A+u3O1mYmQ2aLCKi9QiPvQxYIGke8BzZq83fXXD87cCUvnVJS4CrUqKYAHwPuCYifn6EcRy2Y8c3sH7bnmqd3sxsxKjYTHkR0U32osF7gceBOyNiuaRrJZ1TYvfLgeOBj0t6KH2mVirWwUwf3+iWhZkZ5U1+dNgi4h7gngFlnxik7lkFy38P/H0lYyvHseMbeGHHXnp6g3yunDF9M7PRyXNwD2HahEa6e4PNftbCzMY4J4shTBuXPWuxzl1RZjbGOVkMYdqELFk8v92D3GY2tjlZDMEP5pmZZZwshuAH88zMMk4WQ/CDeWZmGSeLEvxgnpmZk0VJ08c3ss7JwszGOCeLEmZNamL9jr10dnvGPDMbu5wsSpgzuYkIWLvVrQszG7ucLEqYM7kJgGc2d1Q5EjOz6nGyKGH2pGYA1mzeXeVIzMyqx8mihCktdTTV5VmzxS0LMxu7nCxKkMScyc2scTeUmY1hThZlOH5qC09t2FntMMzMqsbJogwLprbw7JY9dHR1VzsUM7OqcLIowwnHtADw9AYPcpvZ2ORkUYbjp2bTkT/5gruizGxscrIow5zJTdTlc04WZjZmOVmUoTaf44RjW1i+bke1QzEzqwonizKdNH08j67bTkRUOxQzs6POyaJMJ80Yz7aOfX5HlJmNSU4WZTppxngAHn1ue5UjMTM7+pwsynTitFbqanL88pmt1Q7FzOyoc7IoU31Nnt+aOZ5lq50szGzscbI4BIvmTuLR57azp8sTIZnZ2OJkcQhOnTuR7t7goWe3VTsUM7OjysniELxq9iQkaF+9pdqhmJkdVU4Wh2B8Uy0vOaaVZWs8bmFmY4uTxSFaNHciv1yzlZ5eP5xnZmNHRZOFpLMlPSFphaSrh6h3vqSQtCitT5Z0n6Rdkm6sZIyH6vTjJrOrs9vjFmY2plQsWUjKAzcBbwEWAhdJWlikXitwBbC0oHgv8HHgqkrFd7hee3wbOcGSJzZUOxQzs6Omki2L04AVEbEyIrqAO4Bzi9S7DrieLEEAEBG7I+JnhWUjxfimWl41ZyJLnthY7VDMzI6aSiaLGcCzBetrU1k/SacAsyLi7grGMezOeslUfv3cdjbsHHG5zMysIiqZLFSkrH9UWFIOuAG48rBPIF0qqV1S+8aNR+8v/bNe0gbAf7t1YWZjRCWTxVpgVsH6TGBdwXorcBKwRNJq4HRgcd8gdzki4uaIWBQRi9ra2oYh5PIsnDaOqa313OdxCzMbIyqZLJYBCyTNk1QHXAgs7tsYEdsjYkpEzI2IucD9wDkR0V7BmIaFJM4+6Vh+/PgGtu/ZV+1wzMwqrmLJIiK6gcuBe4HHgTsjYrmkayWdU2r/1Nr4DHCJpLXF7qSqpvNfNZPO7l4WP7yudGUzsxe5mkoePCLuAe4ZUPaJQeqeNWB9bsUCGwYvnzGehdPGcevPV/Hu02aTzxUbojEzGx38BPdhksSHzprPyo27uXf589UOx8ysopwsjsBbXz6N46Y0c9N9Kzw3t5mNak4WRyCfE5edNZ/l63aw5EnfRmtmo5eTxRE67+QZTBvfwBeWPF3tUMzMKsbJ4gjV1eT449cexwOrtvDgGs9zYWajk5PFMLjw1FlMaKrlpvvcujCz0cnJYhg019dw6ZnH8V+/2cBXl66pdjhmZsPOyWKYfPDM+Zz1kjb+9jvLeWCVu6PMbHRxshgm+Zz43IWnMHtSEx/6jwd5btueaodkZjZsnCyG0fjGWm5+3yK6unu59Cvt7OnqqXZIZmbDwslimB0/tYXPXXQyj63fwV9+62F6PVe3mY0CThYV8PqXHsNfvvkl3P3Ieq785sPs6+mtdkhmZkekoi8SHMs+9Lr59PYG//zDJ9myu4svXPxKmup8uc3sxcktiwqRxOWvX8Cn3/FyfvrURt7xL/+P3zy/o9phmZkdFieLCrvwtNnc+oHT2LSrk3Nu/Dn/9rNVHscwsxcdJ4uj4HUntPGDj5zJmQumcN3dj/HeW5by5As7qx2WmVnZnCyOkikt9Xz5fYv41NtP4pG12zn7sz/ho996hOe37612aGZmJWm0zMOwaNGiaG8f8dN3A7B1dxc33reC23+xhlwO3vmqWbz/jDkcP7W12qGZ2Rgj6cGIWFSynpNF9Ty7pYPP//gpvvPQOrp6evmd46dw8emzeeOJx1CTd6PPzCrPyeJFZPOuTr7+wDN8bekzrNu+l2PG1fOuU2dz4amzmD6hsdrhmdko5mTxItTd08uSJzby1aVrWPLkRgS84cRjeM+rZ3PmgjZyOVU7RDMbZcpNFn5KbASpyed448JjeOPCY3h2Swdff+AZ7mx/lh899gLjG2tZNGcip82bxKnzJnHS9PHU1birysyODrcsRriu7l5+9NgL/PSpjTywagsrN+0GoKE2xymzJrJo7kROmT2BRXMnMa6htsrRmtmLjbuhRqmNOztpX72FB1ZvYdnqLTy+fic9vUFOcOK0cbxi5gReMXM8rzluMnMmNyG568rMBudkMUZ0dHXz0LPbuH/lFn65ZiuPrN3Gjr3dALTU13D81BYWTG1hwTEtLJjayoJjWpgxodFJxMwAj1mMGU11NZwxfwpnzJ8CQESwctNufvH0Zp56YSdPbdjFkic38s0H1/bvM7GplpNmjOelx7YyZ3IzcyY3MXdyM9MnNJL3ILqZFeFkMcpIYn5bC/PbWg4o39bRxYoNu3j8+Z0sf247j6zdzld+sYbO7v2vT6/Ni1kTm5g9uYnZk7LPnMnNzJ3cxKxJTTTU5o/2P8fMRggnizFiQlMdi+ZOYtHcSf1lvb3Bhp2drN68mzWbd7N6cwerN+3mmS0dPLh6Kzs7u/vrSnDsuAZmTWpizqQm5k5pZvakJo4d38Dk5jomN9czrrHG3Vtmo5STxRiWy4ljxzdw7PgGTj9u8gHbIoJtHftYvTlLHqs3dbBmy26e2dzBkic3srGgW6tPbV60tdQzbUIjbS31TGmtY0pLPceMa2DhtHHMndzshGL2IuVkYUVJYmJzHROb6zhl9sSDtu/u7OaZLR1s3NnJ5t2dbN7VxebdXbywYy/rt+1lxcZd3L+qk20d+w7YryaXHXdycx1TxzUwtbWettZ6JjVl55rYVJu+ZsvjGmr9MKLZCOBkYYelub6GE6eN48RpQ9fb19PL89v38sja7azfvoetHV1s2d3Fxp1dbNi5lyef38mmXZ10DzLHh5Td1TWuoZbWhhrGN9Yyf2oLrfU1NNXV0Fyfp7m+hub6Glrq8zTX1dDSUENLf1kN9TU5t2bMjlBFk4Wks4HPAXngXyPi04PUOx/4JnBqRLSnsmuAPwJ6gCsi4t5KxmqVUZvPMWtSNkA+mIhgx95utnV0sbVjH1s7uti6O1ve3tHFjr3d7Ni7j517u9m0q5MfPPo8uzu7DxicH0pNTv2JI0sieVoaagdNLn3LzfV5Wutrs/r1Wb3G2rwTj41JFUsWkvLATcCbgLXAMkmLI+KxAfVagSuApQVlC4ELgZcB04H/K+mEiOipVLxWPZIY31jL+MZa5kwuXb9Pd08vHft66OjsYVfnPnZ19rC7s5tdnd3s2tvN7q6C5c5udqV6uzt72L5nH+u27cnK93azq6ubch45yon+BNNYl6epLk9T7f7lxro8jbXpU5enLp+jJp+jJidq8qImJ/K5XP9y/7b+7blURwP2O3iffE7U5nLk+8pTmZOZVUIlWxanASsiYiWApDuAc4HHBtS7DrgeuKqg7FzgjojoBFZJWpGO94sKxmsvMjX5HOPyufSak4YjOlZEsGdfT5Y4OrvZ3dnDzpRYDkhEafuuzm72dPXQ0dVNR1cPWzu6eG5bD3v3ZZ89XT107OspKwENt8IEk8+J2nxKPoUJp2D5wMRUbvI6MGnlJHLKEn9OIp+DnJTWOWh7X5lE//4SKeZsW00+K6/J5cjlIF9Qls/t/9Tk9pcVfu07nwac/4B1Dt7e99UOVMlkMQN4tmB9LfDqwgqSTgFmRcTdkq4asO/9A/adMfAEki4FLgWYPXv2MIVtY5EkmuqycZCpw3TMiKC7N+jpDfb19Kav2Xp3by/dPdn2wuWe3t6COkF3T2/6ur9eT2+wrzc7Xn95/3K2z/5zDdjWv35wPF3dvXR09QyIZ0CsBfH0xTFKXgJxgGIJJK/9SS2XOzDpHVA3Jbu8BCnniJSo0rGzMu1fLkhOGhDHwOW+8+Vz2TlyOThj/hSueMOCSl0OoLLJolhq7v+2kpQDbgAuOdR9+wsibgZuhux1H4cVpVmFSNlf9bV5RvUDjb29QQC9Edmnt2A5sqTZG/vLon8527c3ssTTV6e7Z39Zd9peWNbTe+C2wrKeiHRMCPafv/CcB62T1gtiODDm9LXI9p6+5d4BdVNckH5xFZwHIArWC39xHZh446DyvuvcU3jdeuk/VyVVMlmsBWYVrM8E1hWstwInAUtSVj0WWCzpnDL2NbMRou/W5nzRv/FstKjkhAjLgAWS5kmqIxuwXty3MSK2R8SUiJgbEXPJup3OSXdDLQYulFQvaR6wAHiggrGamdkQKtayiIhuSZcD95LdOntLRCyXdC3QHhGLh9h3uaQ7yQbDu4EP+04oM7Pq8SvKzczW8Z/+AAAHnUlEQVTGsHJfUe55Oc3MrCQnCzMzK8nJwszMSnKyMDOzkpwszMyspFFzN5SkjcCaIzjEFGDTMIVTSY5zeDnO4eU4h9fRiHNORLSVqjRqksWRktRezu1j1eY4h5fjHF6Oc3iNpDjdDWVmZiU5WZiZWUlOFvvdXO0AyuQ4h5fjHF6Oc3iNmDg9ZmFmZiW5ZWFmZiU5WZiZWUljPllIOlvSE5JWSLq62vEUkrRa0q8lPSSpPZVNkvQjSU+lrxOrENctkjZIerSgrGhcynw+Xd9HJL2yynF+UtJz6Zo+JOmtBduuSXE+IenNRzHOWZLuk/S4pOWS/iyVj6hrOkScI+qaSmqQ9ICkh1Ocf5fK50lamq7nN9I8O6R5c76R4lwqaW6V47xN0qqC63lyKq/azxLQN8Xg2PyQzbPxNHAcUAc8DCysdlwF8a0Gpgwoux64Oi1fDfxTFeI6E3gl8GipuIC3At8nmyr3dGBpleP8JHBVkboL0/9/PTAvfV/kj1Kc04BXpuVW4MkUz4i6pkPEOaKuabouLWm5FliartOdwIWp/IvAh9LynwJfTMsXAt84StdzsDhvA84vUr9qP0sRMeZbFqcBKyJiZUR0AXcA51Y5plLOBf49Lf87cN7RDiAifgJsGVA8WFznAl+JzP3ABEnTqhjnYM4F7oiIzohYBawg+/6ouIhYHxG/TMs7gceBGYywazpEnIOpyjVN12VXWq1NnwBeD3wrlQ+8nn3X+VvAGyRVfI7YIeIcTNV+lsDdUDOAZwvW1zL0N//RFsAPJT0o6dJUdkxErIfshxeYWrXoDjRYXCPxGl+emvG3FHTjjYg4UxfIKWR/ZY7YazogThhh11RSXtJDwAbgR2Stmm0R0V0klv440/btwORqxBkRfdfzU+l63iCpfmCcyVH9fx/ryaLYXw8j6V7i346IVwJvAT4s6cxqB3QYRto1/gIwHzgZWA/8r1Re9TgltQB3AR+JiB1DVS1SdtRiLRLniLumEdETEScDM8laMycOEcuIiVPSScA1wEuBU4FJwEerHSc4WawFZhWszwTWVSmWg0TEuvR1A/Btsm/6F/qanunrhupFeIDB4hpR1zgiXkg/oL3Al9nfLVLVOCXVkv0C/mpE/GcqHnHXtFicI/Wapti2AUvI+vgnSKopEkt/nGn7eMrvvhzuOM9O3X0REZ3ArYyQ6znWk8UyYEG6S6KObHBrcZVjAkBSs6TWvmXgd4FHyeJ7f6r2fuA71YnwIIPFtRh4X7qT43Rge1/XSjUM6ON9O9k1hSzOC9OdMfOABcADRykmAf8GPB4RnynYNKKu6WBxjrRrKqlN0oS03Ai8kWx85T7g/FRt4PXsu87nA/8VaUS5CnH+puAPBJGNqxRez+r9LB3N0fSR+CG7w+BJsj7Nv652PAVxHUd2J8nDwPK+2Mj6Un8MPJW+TqpCbF8n627YR/bXzh8NFhdZ0/mmdH1/DSyqcpy3pzgeIfvhm1ZQ/69TnE8AbzmKcf4OWXfCI8BD6fPWkXZNh4hzRF1T4BXAr1I8jwKfSOXHkSWrFcA3gfpU3pDWV6Ttx1U5zv9K1/NR4D/Yf8dU1X6WIsKv+zAzs9LGejeUmZmVwcnCzMxKcrIwM7OSnCzMzKwkJwszMyvJycLMzEpysjA7ApJOHvBK7nM0TK+6l/QRSU3DcSyzI+XnLMyOgKRLyB6OurwCx16djr3pEPbJR0TPcMdi5paFjQmS5iqbtOfLaaKZH6ZXLBSrO1/SD9Lbfn8q6aWp/J2SHk2T1fwkvSLmWuBdaZKad0m6RNKNqf5tkr6gbMKglZJel97K+rik2wrO9wVJ7TpwApwrgOnAfZLuS2UXKZsM61FJ/1Sw/y5J10paCrxG0qclPZbeWvrPlbmiNuYczcfF/fGnWh9gLtANnJzW7wQuHqTuj4EFafnVZO8KguwVCzPS8oT09RLgxoJ9+9fJJrG5g+w1DecCO4CXk/2R9mBBLH2v8ciTvUzuFWl9NWnyK7LE8QzQBtSQvRLivLQtgAv6jkX2ag0VxumPP0f6ccvCxpJVEfFQWn6QLIEcIL1++wzgm2megS+RzRAH8HPgNkl/QvaLvRzfjYggSzQvRMSvI3s76/KC818g6Zdk7wl6GdkMcwOdCiyJiI2RzbnwVbKZAAF6yN4EC1lC2gv8q6R3AB1lxmk2pJrSVcxGjc6C5R6gWDdUjmySnJMHboiIyyS9Gvg9oH9u5DLP2Tvg/L1ATXob61XAqRGxNXVPNRQ5zlAzt+2NNE4REd2STgPeQPYW5cvJZogzOyJuWZgViGwyn1WS3gnZa6Il/VZanh8RSyPiE8AmsrkFdpLNR324xgG7ge2SjiGb6KpP4bGXAq+TNEVSHrgI+O+BB0sto/ERcQ/wEbIJicyOmFsWZgd7D/AFSX9DNi/yHWSviv+fkhaQ/ZX/41T2DHB16rL6x0M9UUQ8LOlXZN1SK8m6uvrcDHxf0vqI+B+SriGbk0HAPRFRbC6TVuA7khpSvT8/1JjMivGts2ZmVpK7oczMrCR3Q9mYJekm4LcHFH8uIm6tRjxmI5m7oczMrCR3Q5mZWUlOFmZmVpKThZmZleRkYWZmJf1/1AtD5buEhpEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2a695050ef0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "### cv_result = pd.DataFrame.from_csv('lgbm1_nestimators.csv')\n",
    "cv_result = pd.read_json(\"lgbm_1.json\")\n",
    "\n",
    "# plot\n",
    "test_means = cv_result['binary_logloss-mean']\n",
    "#train_means = cvresult['train-logloss-mean']\n",
    "\n",
    "x_axis = range(0, cv_result.shape[0])\n",
    "pyplot.plot(x_axis, test_means) \n",
    "pyplot.title(\"binary_logloss-mean\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'lgbm1_n_estimators.png')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 2、调整树的层数和叶子节点个数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [6, 7, 8], 'num_leaves': [50, 60, 80]}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_leaves=[50,60,80]\n",
    "max_depth=[6,7,8]\n",
    "param_test2_1=dict(num_leaves=num_leaves,max_depth=max_depth)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1042: UserWarning: categorical_feature in Dataset is overridden. New categorical_feature is [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]\n",
      "  warnings.warn('categorical_feature in Dataset is overridden. New categorical_feature is {}'.format(sorted(list(categorical_feature))))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=0.8,\n",
       "        learning_rate=0.1, max_bin=363, max_depth=6, min_child_samples=100,\n",
       "        min_child_weight=1, min_split_gain=0.0, n_estimators=365,\n",
       "        n_jobs=-1, nthread=6, num_leaves=50, objective='binary',\n",
       "        random_state=None, reg_alpha=0, reg_lambda=0, scale_pos_weight=1,\n",
       "        seed=0, silent=True, subsample=0.8, subsample_for_bin=200000,\n",
       "        subsample_freq=1),\n",
       "       fit_params=None, iid=True, n_jobs=3,\n",
       "       param_grid={'num_leaves': [50, 60, 80], 'max_depth': [6, 7, 8]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=3)\n",
    "\n",
    "params = {'boosting_type': 'gbdt', \n",
    "          'objective': 'binary', \n",
    "          'nthread': 6, \n",
    "          'silent': True,\n",
    "          'learning_rate': 0.1, \n",
    "          'num_leaves': 50, \n",
    "          'max_depth': 6,\n",
    "          'max_bin': 363, \n",
    "          'subsample_for_bin': 200000,#默认200000，用来构建直方图数据数量\n",
    "          'subsample': 0.8, #默认1\n",
    "          'subsample_freq': 1, \n",
    "          'colsample_bytree': 0.8, \n",
    "          'reg_alpha': 0,          \n",
    "          'reg_lambda': 0,  \n",
    "          'min_split_gain': 0.0, \n",
    "          'min_child_weight': 1,         \n",
    "          'min_child_samples': 100, \n",
    "          'scale_pos_weight': 1,\n",
    "         }\n",
    "lgbm2_1=lgbm.sklearn.LGBMClassifier(n_estimators=365,seed=0,**params)\n",
    "gsearch2_1=GridSearchCV(lgbm2_1,param_grid=param_test2_1,scoring='neg_log_loss',n_jobs=3, cv=kfold)\n",
    "\n",
    "\n",
    "gsearch2_1.fit(X_train,y_train,categorical_feature=[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mean_fit_time': array([103.00007288, 132.42437975, 153.99344206, 169.69558167,\n",
      "       172.11771496, 178.8448449 , 177.08994706, 183.67917387,\n",
      "       168.77629352]), 'std_fit_time': array([ 9.30479354, 26.23411794,  3.40672296,  6.32590884,  2.92489557,\n",
      "        3.35738066,  2.24613149,  4.87940454, 18.83478456]), 'mean_score_time': array([21.64225586, 13.17840155, 12.71249477, 14.30360476, 13.93203108,\n",
      "       14.34414442, 14.22060712, 14.85142008, 18.7050755 ]), 'std_score_time': array([6.85382836, 0.66749651, 0.13430911, 0.67895085, 0.47263521,\n",
      "       0.3245092 , 0.25607761, 0.31643679, 4.82394103]), 'param_max_depth': masked_array(data=[6, 6, 6, 7, 7, 7, 8, 8, 8],\n",
      "             mask=[False, False, False, False, False, False, False, False,\n",
      "                   False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_num_leaves': masked_array(data=[50, 60, 80, 50, 60, 80, 50, 60, 80],\n",
      "             mask=[False, False, False, False, False, False, False, False,\n",
      "                   False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'max_depth': 6, 'num_leaves': 50}, {'max_depth': 6, 'num_leaves': 60}, {'max_depth': 6, 'num_leaves': 80}, {'max_depth': 7, 'num_leaves': 50}, {'max_depth': 7, 'num_leaves': 60}, {'max_depth': 7, 'num_leaves': 80}, {'max_depth': 8, 'num_leaves': 50}, {'max_depth': 8, 'num_leaves': 60}, {'max_depth': 8, 'num_leaves': 80}], 'split0_test_score': array([-0.39972166, -0.39965311, -0.39971417, -0.39958501, -0.39958986,\n",
      "       -0.39969027, -0.39955242, -0.39961145, -0.39965136]), 'split1_test_score': array([-0.39964054, -0.39979181, -0.39969156, -0.39955184, -0.39957714,\n",
      "       -0.39965204, -0.39948155, -0.39947828, -0.39969074]), 'split2_test_score': array([-0.39953739, -0.39954156, -0.39954432, -0.39944938, -0.39941897,\n",
      "       -0.39953974, -0.39942667, -0.3993762 , -0.39953964]), 'mean_test_score': array([-0.3996332 , -0.39966216, -0.39965002, -0.39952874, -0.39952866,\n",
      "       -0.39962735, -0.39948688, -0.39948865, -0.39962725]), 'std_test_score': array([7.54089031e-05, 1.02363007e-04, 7.53052901e-05, 5.77291081e-05,\n",
      "       7.77337214e-05, 6.38872297e-05, 5.14733380e-05, 9.63205894e-05,\n",
      "       6.39994369e-05]), 'rank_test_score': array([7, 9, 8, 4, 3, 6, 1, 2, 5]), 'split0_train_score': array([-0.39122959, -0.39089943, -0.39088469, -0.38988339, -0.38894818,\n",
      "       -0.38774804, -0.38918096, -0.38796903, -0.38622647]), 'split1_train_score': array([-0.39115512, -0.39091266, -0.39082423, -0.38984748, -0.38893027,\n",
      "       -0.38758383, -0.38919688, -0.38791591, -0.38609445]), 'split2_train_score': array([-0.39115964, -0.39078501, -0.39073748, -0.38988838, -0.38882565,\n",
      "       -0.3877267 , -0.38924834, -0.38807613, -0.38619787]), 'mean_train_score': array([-0.39118145, -0.3908657 , -0.39081547, -0.38987308, -0.38890137,\n",
      "       -0.38768619, -0.38920873, -0.38798702, -0.38617293]), 'std_train_score': array([3.40930111e-05, 5.73110695e-05, 6.04176392e-05, 1.82189177e-05,\n",
      "       5.40372536e-05, 7.29035759e-05, 2.87571475e-05, 6.66377234e-05,\n",
      "       5.67103938e-05])}\n",
      "-0.3994868801062272\n",
      "logloss of train : 0.39142793253288627\n",
      "logloss of test : 0.4031091261846167\n"
     ]
    }
   ],
   "source": [
    "print(gsearch2_1.cv_results_)\n",
    "print(gsearch2_1.best_score_)\n",
    "\n",
    "y_pred_train = gsearch2_1.predict_proba(X_train)\n",
    "logloss = log_loss(y_train, y_pred_train)\n",
    "print (\"logloss of train :\",logloss )\n",
    "\n",
    "y_pred_test = gsearch2_1.predict_proba(X_test)\n",
    "logloss = log_loss(y_test, y_pred_test)\n",
    "print (\"logloss of test :\",logloss )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.399487 using {'max_depth': 8, 'num_leaves': 50}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8FHW2///XyUYStoRNgbAkTFiNhkUQVwRlSVBUXBBcUIGr4ujo/d2r/mbGmetXv3pn5s4VZxz9SVjUcQSXYcRJVBxURERli2yyJkACyr5KgCzn90dVJ52QjXQ63Z2c5+PBg3R3Vden0pB3qurU+YiqYowxxtRVWKAHYIwxJrRZkBhjjPGJBYkxxhifWJAYY4zxiQWJMcYYn1iQGGOM8YkFiTHGGJ9YkBhjjPGJBYkxxhifRAR6AA2hXbt22r1790APwxhjQsqqVasOqGr7mpZrEkHSvXt3Vq5cGehhGGNMSBGRnbVZzk5tGWOM8YkFiTHGGJ9YkBhjjPGJBYkxxhifWJAYY4zxiQWJMcYYn1iQGGOM8UmTuI+krt7P3g1AQnwMneNi6dCyGWFhEuBRGWNMcLEgqcYfP9nCzoMnSx9HhYfRMS6aznExpeGSEB9D53jn8fmtookIt4M8Y0zTYkFSjayHr2DPkQLyDxeQf6SA/MMn2X3YefzZ5v3sP3663PLhYcL5raKdYPGETXwMCfGxdI6LoWNcNM0iwgO0N8YY4x8WJNVo3iyC5PNaknxey0pfP1VYzJ4jBex2w8YJmZPsPlLA8pyD7D12ihItW14EOrRsVhosnqBxvnaObqIjLWiMMaHFgsQH0ZHhJLVvQVL7FpW+Xlhcwo9HT5HndSSz2z2yWZN3mKx1P1DknTRAuxZRpcHiOWXWOa7syKZFM/vIjDHBxa8/lURkNDADCAcyVPX5Cq/fD0wHioETwDRV3SgiUcD/BwwCSoBHVPVzd51ngbuAeFWt/Cd4kIgMD6NLm1i6tImt9PXiEmXvsVOl4eIdNht/OMYn3+/lTFFJuXVax0SWhkvFsOkSH0urmAhErCDAGNNwRFVrXqoubywSDmwBrgXygRXA7aq60WuZVqp6zP36euBBVR0tItOBQap6j4h0AD4ELlbVEhG5BNgJbK1tkAwaNEhDsftvSYly4KfTXqfNCth95GTp491HCjh5prjcOi2aRZQ7bVaxKKBt8ygLGmNMrYjIKlUdVNNy/jwiGQxsU9Ucd0DzgHFAaZB4QsTVHPCkWl9gsbvMPhE5gnN08q2qfu2+nx+HHhzCwoQOLaPp0DKaAV3jz3pdVTl8srDctZn8w54/J/l2xyGOnyoqt050ZJh7qizW68jG8yeW9i2sxNkYc278GSSdgTyvx/nAkIoLuUcfjwFRwHD36e+AcW74dAEGun9/W9uNi8g0YBpA165d6zD84CcitGkeRZvmUaQktK50maMFhaVHLxVPn63LP8Lhk4Xllo8KD6NTnKfyLNarGMA5orESZ2NMRf4Mksp+rT3rPJqqvgS8JCITgV8BdwOzgT7ASpzTWF8BRRXXrY6qvgq8Cs6prXMaeSPSOiaS1jGR9O3UqtLXfzpdVGWJ86eb91VZ4uxd2uxd6tyxdQxRERY0xjQl/gySfJyjCI8EYE81y88DXgZQ1SLgUc8LIvIVsNUPY2zyfCpx3n6Qvcd2n1XifF7L6HJFABWLAqzE2ZjGxZ9BsgJIFpFEYDcwAZjovYCIJKuqJyDSccNCRGJxCgF+EpFrgSLvi/Sm4fhS4rx612Ey11ZW4tyskps2naKAzvExVuJsTIjx2/9YVS0SkYeAj3HKf2er6gYReRpYqaoLgYdE5BqgEDiMc1oLoAPwsYiU4ITQnZ73FZHf4QRSrIjk45QV/9Zf+2Gq548S57jYshLnim1oEuKsxNmYYOO38t9gEqrlv01BXUqcoyPDiAwLQ8S5ZhMmQliYECYQLoKIuM/jPi/u85Uv7zzGWS7MXV8ov1zp82e/r2fdsvXd9/Vsq7r1vdYtt5xn/BXft/Q9y5avy/6Hh+E1nor77/19qbCdCuM0jVswlP8aU6O6lDjvPXaKohJF1TniKVH3TwkUl36tlKjzWFXd5XCfV4oVr+eddYu0xF3faznPdiq+b4n7vjUuj7s95/nG5twDj8qD3X3trCCsc5BS64AtH+xVB2nNvwhUCOha/oJR4/6HVf4LjIRV8ouQ+31t6JC3IDFBrTYlzqFCPaFTx4DyrFtufe8g9X5cLsCUYvdxzQHra5DiNR7nsdZl/F6/GBSVlHC6qOL6nu9nNeF9DttsbLxDaO1vRvq9wMWCxJgG4jllFo5ghWvBpaqwqxiyWouw9ucvCZWPs/pfEiIa4AZjCxJjTJMnIkSE2zWfurI7x4wxxvjEgsQYY4xPLEiMMcb4xILEGGOMTyxIjDHG+MSCxBhjjE8sSIwxxvjEgsQYY4xPLEiMMcb4xILEGGOMTyxIjDHG+MSCxBhjjE8sSIwxxvjEgsQYY4xPLEiMMcb4xILEGGOMTyxIjDHG+MSCxBhjjE8sSIwxxvjEgsQYY4xPLEiMMcb4xILEGGOMTyxIjDHG+MSCxBhjjE8sSIwxxvjEgsQYY4xP/BokIjJaRDaLyDYReaKS1+8XkXUiki0iX4pIX/f5KBGZ4772nYgM81pnoPv8NhF5UUTEn/tgjDGmen4LEhEJB14CxgB9gds9QeHlb6qaoqqpwO+AP7rPTwVQ1RTgWuB/RMQz1peBaUCy+2e0v/bBGGNMzfx5RDIY2KaqOap6BpgHjPNeQFWPeT1sDqj7dV9gsbvMPuAIMEhEOgKtVHW5qirwOnCDH/fBGGNMDfwZJJ2BPK/H+e5z5YjIdBHZjnNE8rD79HfAOBGJEJFEYCDQxV0/v6b3NMYY03D8GSSVXbvQs55QfUlVewCPA79yn56NExIrgReAr4Ci2r4ngIhME5GVIrJy//79dRi+McaY2vBnkOTjHEV4JAB7qll+Hu5pKlUtUtVHVTVVVccBccBW9z0TavOeqvqqqg5S1UHt27f3YTeMMcZUx59BsgJIFpFEEYkCJgALvRcQkWSvh+k4YYGIxIpIc/fra4EiVd2oqj8Ax0XkErda6y7gfT/ugzHGmBpE+OuNVbVIRB4CPgbCgdmqukFEngZWqupC4CERuQYoBA4Dd7urdwA+FpESYDdwp9dbPwDMBWKAD90/xhhjAkSc4qfGbdCgQbpy5cpzX/HdeyEyFvrfAV2GgN2yYoxpQkRklaoOqmk5vx2RhDxViGoB696FNW9Amx6QOhEuuh1aW6GYMcZ42BFJTU6fgI3vQ/bfYOeXIGGQdDX0nwS90iEyun4Ha4wxQaK2RyQWJOfiUA5kvwXfvQVH8yC6NVxwsxMqnQbYqS9jTKNiQeKl3oLEo6QEdnwBa96E7xdC0Slo38c59XXhbdDyvPrbljHGBIgFiZd6DxJvp47C+r87p77yvwUJh+SRzlFK8iiIiPLPdo0xxs/sYntDiW4Ng+5x/uzfAtlvwnfzYMuHENsWUm51QuX8lECP1Bhj/MKOSPyhuAi2f+qEyuYsKD4D51/olBGn3AKxbRpuLMYYU0d2astLgweJt5OHnBLi7L/CD99BWCT0GuOESo8REG4HhcaY4GRB4iWgQeLtx/XOtZS18+HkAWhxnnNxvv8d0L5XoEdnjDHlWJB4CZog8Sg6A1sXOaGy9WMoKYLOg5xrKf1ugpi4QI/QGGMsSLwFXZB4O7EP1r7tXE/ZtxEioqH3WCdUEq+CsPBAj9AY00RZkHgJ6iDxUIU9a5xAWfeOU1bcKgFSb3fasrTtEegRGmOaGAsSLyERJN4KTznVXtlvOtVfWgJdL3WOUvqOg2YtAz1CY0wTYEHiJeSCxNuxPc59KdlvwsFtENncCZP+k6DbZdaWxRjjNxYkXkI6SDxUIe9bp4x4/QI4cxziu0PqJLhoAsR1DfQIjTGNjAWJl0YRJN7OnITvP3BCJfcLQCDxSqeMuPdYiIoN9AiNMY2AtUhpzKJi4aLbnD+HdzrdiLPfhL9PhWatoN+NTqgkXGynvowxfmdHJI1FSQnsXOYEysb3ofAktOvpdiSeAK06BnqExpgQY6e2vDSJIPF2+jhs+IcTKruWO5Nx9RjhTsaVBhHNAj1CY0wIsCDx0uSCxNvB7WUdiY/thug4p3Fk/0nQMdVOfRljqmRB4qVJB4lHSTHkfO6Eyvf/hOLT0KGfEygpt0KL9oEeoTEmyFiQeLEgqaDgMKx/z+n1tXsVhEU4k3D1n+RMyhUeGegRGmOCgFVtmarFxMPFU5w/+753T33Nh82Z0Ly905E4dSKc1y/QIzXGhAA7IjGO4iLY9i/n3pTNH0FJoXMNpf8dcMF4m4zLmCbITm15sSA5Rz8dhHVvw5o3Ye86CI9yqr363wE9hltHYmOaCAsSL3UNktnrZ9M+pj1jEscQEdZEzwL+sNY59bX2bSg4BC07Oi1ZUidBu+RAj84Y40cWJF7qEiTFJcVMyprEhoMb6NyiM/f0u4cbkm+gWXgTvQej6DRs+cg5Stn2L9BiSBhcNhlXdKtAj9AYU8/qLUhEpAeQr6qnRWQYcCHwuqoeqZeRNoC6HpGUaAlL8paQsS6DtQfW0ja6LXf1u4tbe95Ki6gWfhhpiDj+ozNd8Jo34cBmiIiBPtc5odL9SggLC/QIjTH1oD6DJBsYBHQHPgYWAr1UNa0extkgfL1Goqqs+HEFGesyWP7DclpGtWRi74lM6jOJ+Oj4ehxpiFGF3audC/Tr3oPTR6F1V2cyrtSJTndiY0zIqs8gWa2qA0TkP4BTqvonEVmjqv3ra7D+Vp8X29cfWE/GugwW71pMTEQM45PHc3e/uzm/+fn18v4hq7AANmXCmr86Nz6i0O3yssm4opoHeoTGmHNUn0HyDfAC8EvgOlXNFZH1qnpB/QzV//xRtbX9yHZmr59NZk4mIsL1Pa7n3gvupVurbvW6nZB0NN/tSPw3OJQDUS2g3w3OBfquQ60tizEhoj6DpC9wP7BcVd8SkUTgNlV9vhaDGA3MAMKBjIrriMj9wHSgGDgBTFPVjSISCWQAA3BumnxdVZ9z13kEmAoIMFNVX6hpHP4s/919Yjdz18/l71v/TpEWMbLbSO5LuY/ebXr7ZXshRdVpGpn9ptNE8swJaJPknPa66HZonRDoERpjquGXqi0RiQe6qOraWiwbDmwBrgXygRXA7aq60WuZVqp6zP36euBBVR0tIhOB61V1gojEAhuBYUALYB4wGDgDfAQ8oKpbqxtLQ9xHcqDgAG9sfIP5m+fzU+FPXNH5CqZeOJX+HULmDKB/nT4B3y90LtDv/BIQ6HG1c5TSOx0iYwI9QmNMBbUNkhrLa0TkcxFpJSJtgO+AOSLyx1qMYTCwTVVzVPUMTgCM817AEyKu5oAn1RRoLiIRQAxOaBwD+gBfq+pJVS0ClgA31mIsftcuph2PDnyURTcv4uf9f876A+u568O7uPvDu/ly95c0hTLrajVr4RyJ3JMJD2fDVf8JB7bCe/fBH3rBPx+F/FXOUYwxJqTU5tTWGlXtLyJTcI5GfiMia1X1whrWuxkYrapT3Md3AkNU9aEKy00HHgOigOGqutU9tfUGMAKIBR5V1VdFpA/wPjAUKAAWAytV9eeVbH8aMA2ga9euA3fu3FnjN6M+nSw8yYJtC5izfg57T+6lT5s+3JdyH9d0vYZwuzPcUVICO75wjlK+XwhFp6B977LJuFqeF+gRGtOk1ec1knXASOA14JequqKWQXILMKpCkAyu7Ie++/pEd/m7ReQy4EFgMhAPLAXGqGqOiNyHc13lBM4prwJVfbS6sQSyRUphcSH/zPkns9fPZsexHXRv1Z17L7iXsUljibQuu2VOHYUNC5xQyf8WJBySr3VCpecYiIgK9AiNaXLq7dQW8DTO/SPb3RBJAqq9JuHKB7p4PU4A9lSz/DzgBvfricBHqlqoqvuAZTj3sqCqs1R1gKpeCRyq5VgCJjI8khuTb+Qf4/7BH676A9ER0Tz11VOkLUjjze/fpKCoINBDDA7RrWHgZJjyCUxfAZc9DD98B2/fBf/TCz583GnXYowJOn5rkeJe39iCc3pqN87F9omqusFrmWTPhXIRuQ74jaoOEpHHgd7AvTintlYAE1R1rYh0UNV9ItIVWAQMVdXD1Y0lmJo2qirL9ixj5tqZrN63mvhm8dzR9w4m9J5AqyhrM1JOcRHkfObcm7I5C4rPwPkpzmmvpKugQ19rIGmMH9Xnqa0E4E/AZTgXwb8EHlHV/FoMIg3nHpRwYLaqPisiT+Nc11goIjOAa4BC4DDwkKpuEJEWwBygL06Z7xxV/b37nkuBtu46j6nq4prGEUxB4m313tVkrMtg6e6lNI9szoReE7ij7x20i2kX6KEFn5OHnMm41vwVfsh2nmvWGroMhm5DnftTOg2AyOjAjtOYRqQ+g+QT4G84F78B7gAmqeq1Po+ygQRrkHh8f/B7Zq2fxaIdi4gKj+LGn93IPRfcQ6cWnQI9tOB0eKdzf8qu5bBzudPvC5x2950GlAVLlyEQExfYsRoTwuq115aqptb0XDAL9iDx2HF0B3M2zGHh9oWgkJaUxn0X3EdSXFKghxbcfjoIeV+XBcsP2VBSBIhz+ssTLF2HQuvOgR6tMSGjPoPkX8Bc4C33qduBe1R1hK+DbCihEiQeP/70I69teI13t7zL6eLTjOg6gikpU+jXzqa+rZUzJ2H3Stj1Nez8CvJXOHfVg9NUsttQ6HoJdL0U2vW0bsXGVKE+g6Qr8GecezcU+Ap4WFV31cdAG0KoBYnHoVOHePP7N3lr01scP3OcoR2HMvXCqQw6bxBi/apqr7jImenREyy7voaf9jmvxbRxQ8UNlo4XWamxMS6/TmwlIr+oTY+rYBGqQeJx4swJ3t7yNq9veJ2Dpw5yYfsLmZoylSsTriRM7Lfpc6bqNJP0nArbtRwObXdei4iBhEFusAx1LuY3axnY8RoTIP4Okl2q2rVOIwuAUA8Sj1NFp3h/2/vM2TCH3Sd2kxyfzH0X3Meo7qOa7lTA9eX4Xuc6iydYflwLWgIS5pQcd720LFzsjnvTRPg7SPJUtUvNSwaHxhIkHoUlhXyU+xGz1s1i+9HtJLRI4N6UexnXYxxR4XZapl6cPu5cW/EES/5K8Nw82iapLFi6Xeo8tlONphGyIxIvjS1IPEq0hM/yPiNjbQbrD66nfUx77u53N7f0vIXYyNhAD69xKTrjHKV4rrHsWg4Fh5zXmncoO1rpNhTOS4FwO0I0oc/nIBGR45R14y33EhCjqiHzP6WxBomHqvLNj9+QsTaDb378hlZRrZjUZxITe08kLtruo/CLkhI4uNUrWL6CI279SVQLSLi4LFg6D4IoC3YTevx6RBJqGnuQeFu7fy0Z6zL4LO8zYiJiuLXnrdzV7y46xHYI9NAav6O73Rsl3SOWvRsAhbAI6JhadiqsyyXQvG2gR2tMjSxIvDSlIPHYengrs9bP4qPcjwiTMMb9bBz39ruXLq1C5tJW6Cs4Annflt2Fv3uV0y8MoF2vsmDpegnEdbPrLCboWJB4aYpB4pF3PI+56+eyYNsCirWYUd1HMSVlCj3jewZ6aE1P4SnYs8Y5Dbbra9j1DZw+6rzWslP5YLGGlCYIWJB4acpB4rH/5P7SqYBPFp1kWMIw7ku5j9QOIdPppvEpKYZ935fvG3bcnWnBGlKaIGBB4sWCpMzR00f526a/8eb3b3L09FEuPv9ipqRMYWjHoXa3fKCpOhfsrSGlCRL12SKlsuqto8BK4N9VNafOo2wgFiRnO1l4kne3vMtrG15jX8E++rbty9SUqQzvOtzulg8m1TWkPK9fWdmxNaQ0flCfQfJfODMb/g2n9HcCcD6wGXhAVYf5PFo/syCp2pniM3yw/QNmrZ9F3vE8klonce8F95KWlEZkmE0FHHSqa0gZ17UsVLoOhfa97AK+8Ul9Bsk3qjqkwnNfq+olIvKdql7k41j9zoKkZkUlRXyy8xMy1mWw5fAWOjbvyD0X3MONP7uR6Ag7Nx+0atWQ0g0Wa0hpzlF9Bsly4H+Bd92nbsaZmfCSUJmXxIKk9lSVpbuXMnPtTLL3Z9Mmug139r2T23rdRssoa14Y9GrVkNJto28NKU0N6jNIkoAZOG3kAZYDj+LMwz5QVb/0cax+Z0Fy7lSVVXtXkbEug2V7ltEysiUTek9gUp9JtI2xm+lCijWkNHVkVVteLEh8s+HgBmatm8W/dv6LZuHNGN9zPJP7Teb85ucHemimLqwhpaml+jwiSQD+BFyGU731JfCIqubXx0AbggVJ/cg5msPsdbPJzMkEYGyPsdx7wb0ktk4M8MiMT6whpalCfQbJJzgVW2+4T90BTFLVa30eZQOxIKlfP5z4gbkb5vLe1vc4U3yGa7pdw5SUKfRt2zfQQzP1wRpSGld9BslZF9RD5SK7hwWJfxwsOFg6FfCJwhNc1vkypqZMZeB5AwM9NFPfrCFlk1SfQfIvYC7wlvvU7cA9qjrC10E2FAsS/zp+5jjzN8/njY1vcOjUIfp36M+UlClc0fkKu1u+saqpIWU3r/tZ4rradZYQVZ9B0hX4M07VlgJfAQ+r6q76GGhDsCBpGAVFBSzYuoC5G+byw08/0Cu+F1NSpnBtt2sJtwaEjVtNDSm9g6VDH2tIGSL8PUPiL1T1hTqNLAAsSBpWYUkhWTlZzFo/i9yjuXRt2ZV7L7iX63pcZ1MBNxU1NaTsOsS9iH8pdOpvDSmDlE2168WCJDBKtIRPd33KzHUz2XhwIx1iOzC532TGJ4+3qYCbmtKGlF+XHbXs3+S8Fh4FnQeWBUuXwdaQMkj4O0jyVDVkZkiyIAksVWX5nuXMXDeTlXtXEtcsjkl9JnF779tp3ax1oIdnAuWng5D3TVmw7FljDSmDjB2ReLEgCR7Z+7LJWJfBkvwlxEbEclvv27ir7120i2kX6KGZQLOGlEHH5yCpon08OB2AY1Q1ZO5KsiAJPpsPbWbW+ll8vONjIiSCG5NvZHK/ySS0TAj00EywsIaUAWctUrxYkASvXcd2MXv9bBZuX0iJljAmcQz3XXAfP4v/WaCHZoKNNaRscEERJCIyGqfhYziQoarPV3j9fmA6UAycAKap6kYRiQQygAFABPC6qj7nrvMoMAXnaGkdzj0tp6obhwVJ8Nv7015e3/g672x5h4KiAoZ3Gc6UlCmktE8J9NBMMKtNQ0pP6XGLDoEebcgJeJCISDiwBbgWyAdWALer6kavZVqp6jH36+uBB1V1tIhMBK5X1QkiEgtsBIYBhTi9vvqqaoGIvA1kqerc6sZiQRI6jpw6UjoV8LEzxxjScQhTUqYw5PwhdnOjqVm1DSl7lLV26TrUGlLWQm2DxJ/XOQYD2zxT8YrIPGAcTigA4AkRV3PKrsko0FxEIoAY4AxwzP06AogRkUIgFmf2RtNIxEXH8WDqg9zd727e2fwOr298namLppLSLoUpKVMY1mWYTQVsqtasJfQY7vyBsxtSbs6C7L86r3kaUnZzux1bQ8o68+cRyc3AaFWd4j6+Exiiqg9VWG468BgQBQxX1a3uqa03gBE4YfGoqr7qLv8I8CxQACxS1Uk1jcWOSELX6eLTvL/tfWavn83uE7v5WdzPuOeCe7i227XERMQEengm1NSmIaUnWKwhZVCc2roFGFUhSAar6s+rWH6iu/zdInIZ8CAwGYgHlgJjgMPAe8BtwBHgHeBdVf1rJe83DZgG0LVr14E7d+6s3x00DaqopIiPd3xMxroMth3ZRmxELCO6jiAtKY1LOl5CRJj9JmnqqKaGlN7tXWLbBHq0DSoYgmQo8FtVHeU+fhLAc9G8kuXDgMOq2lpEXgK+VtU33NdmAx/hnPIarar3uc/fBVyiqg9WNxY7Imk8SrSElT+uJCs3i0U7F3H8zHHaRLdhVPdRpCWmcVH7i+xaivFNdQ0p2/cuuwO/6yWNviFlMARJBM7F9hE40/KuACaq6gavZZJVdav79XXAb1R1kIg8DvQG7sU5tbUCmIBzjWQ2cDHOqa25wEpV/VN1Y7EgaZzOFJ9h6e6lZOVksSR/CaeLT9O5RWfSEtNIT0qnR1yPQA/RNAalDSndYPFuSNmqc/n7WTr0hbDGcw0v4EHiDiINeAGn/He2qj4rIk/j/PBfKCIzgGtwqrEOAw+p6gYRaQHMAfri3AA5R1V/777nf+Gc2ioC1gBTVPV0deOwIGn8Tpw5weJdi8nKzeLrH76mREvoFd+L9KR0xiSOsWmBTf0pKYF9G2toSOkGS+cBENEssOP1QVAESbCwIGlaDhQc4OMdH5OVk8XaA2sBGHjeQNKT0hnZbaT19zL1q9qGlM2cMPEES4g1pLQg8WJB0nTlHcsjMzeTzJxMdhzbQURYBJd3upz0pHSu6nKVVX4Z/6ixIeXQstLjVp0CPdoqWZB4sSAxqsqmQ5vIzMnkw9wP2Vewj5iIGEZ0HUF6UrpVfhn/qrEh5aVlwdKuZ9BcwLcg8WJBYrwVlxSzet9qMnMyy1V+jew2kvSkdKv8Mv5XY0NKryOWjhdBeGRAhmlB4sWCxFTlTPEZvtz9JZk5mVb5ZQKntg0puw11bppsoIaUFiReLEhMbZw4c4JP8z4lMyezXOVXWlIaaYlpVvllGlaVDSnD3YaUQ/3ekNKCxIsFiTlXVVV+pSWmMar7KKv8Mg2vpoaU3nfg11NDSgsSLxYkxhd5x/LIys0iMzeT3KO5pZVfaUlpDOsyzCq/TGBUbEi5azkUHHJea3Fe2Y2S/e+EZi3qtAkLEi8WJKY+eCq/snKzyMrNYt/JssqvtMQ0Lul0CZFhgbkoakylDSmP/whP5EFkdJ3e0oLEiwWJqW9W+WVCwk8HoHm7Oq9uQeLFgsT4k6fyKys3i8/zPi9X+ZWWmGbTBpuQZUHixYLENBRP5VdWThbLf1hervJrTPcxdGzRMdBDNKbWLEi8WJCYQCit/MrNYu3+8pVfI7uNJC46dHoumabJgsSLBYkJtOoqv65KuIrYyKY9E58JThYkXixITLCoqvJreNfhpCemW+WXCSoWJF4sSEwwKtESVu1dVa7yK75ZPCPAyFK9AAAaOElEQVS7j2Rs0lir/DIBZ0HixYLEBLuqKr/GJI4hPTHdKr9MQFiQeLEgMaHkp8KfnNkevSq/esb3LC0ntsov01AsSLxYkJhQVVnl14AOA0pne7TKL+NPFiReLEhMY5B3PI+sHK/KL4ngss6XkZbo9Pyyyi9T3yxIvFiQmMZEVdl8eDOZOZlnVX6lJaYxtNNQq/wy9cKCxIsFiWmsqqv8Sk9KJ7V9qlV+mTqzIPFSWZAUFhaSn5/PqVOnAjQqE0jR0dEkJCQQGdl4fnM/U3yGZbuXkZmbeVblV1piGsnxyYEeogkxFiReKguS3NxcWrZsSdu2be03tiZGVTl48CDHjx8nMTEx0MPxi58Kf+LTXWWzPRZrsVV+mXNmQeKlsiD5/vvv6d27t4VIE6WqbNq0iT59+gR6KH53oOAAi3YsIjM386zKr2u7XUt8dHyAR2iClQWJl6qCpCn8EDFVa4r/BvKO5/Fh7odk5mSSczTHKr9MtWobJBENMRhjTHDo0rIL0y6cxtSUqWw+vJmsHKfn15L8JVb5ZeosLNADaKqOHDnCX/7yl3NeLy0tjSNHjvhhRP7x29/+lj/84Q91Wjc7O5usrKxzfi9V5Ze//CU9e/akT58+vPjii3XafmMmIvRu05vHBj3GopsXMXvUbNKT0lmav5Tpi6cz4u0RPPP1M6zZt4YSLQn0cE2QsyOSAPEEyYMPPlju+eLiYsLDw6tcz/sHa2OXnZ3NypUrSUtLO6f15s6dS15eHps2bSIsLIx9+/b5aYSNQ5iEcfH5F3Px+Rfz5OAnWbZ7GVm5Wfxj2z+Yv3k+nZp3Ii0pzSq/TJXsiCRAnnjiCbZv305qaioXX3wxV199NRMnTiQlJQWAG264gYEDB9KvXz9effXV0vW6d+/OgQMH2LFjB3369GHq1Kn069ePkSNHUlBQUOX2hg0bxqOPPsqVV15Jnz59WLFiBTfddBPJycn86le/Kl2usu3u3LmT5ORkDhw4QElJCVdccQWLFi2qclvPPvssvXr14pprrmHz5s2lz2/fvp3Ro0czcOBArrjiCjZt2gTA5MmTuf/++7niiivo2bMn//znPzlz5gxPPfUU8+fPJzU1lfnz5wOwceNGhg0bRlJSUpVHGi+//DJPPfUUYWHOP+8OHTpU+1mYMlHhUVzd9Wp+f9XvWXLbEv7v5f+XxLhE5qyfw00Lb+KmhTcxa90s9pzYE+ihmiBiRyTAf32wgY17jtXre/bt1IrfXNevyteff/551q9fT3Z2Np9//jnp6emsX7++tBx19uzZtGnThoKCAi6++GLGjx9P27Zty73H1q1beeutt5g5cya33nor7733HnfccUeV24yKiuKLL75gxowZjBs3jlWrVtGmTRt69OjBo48+Stu2bSvdbrdu3Xj88ce5//77GTJkCH379mXkyJGVbmPVqlXMmzePNWvWUFRUxIABAxg4cCAA06ZN45VXXiE5OZlvvvmGBx98kE8//RSAHTt2sGTJErZv387VV1/Ntm3bePrpp1m5ciV//vOfAefU1qZNm/jss884fvw4vXr14oEHHiAyMpK0tDQyMjLo1KkT27dvZ/78+SxYsID27dvz4osvkpxsv0mfq+aRzbmux3Vc1+O60sqvrNwsXlj9Ai+sfsEqv0wpC5IgMXjw4HL3NLz44ossWLAAgLy8PLZu3XpWkCQmJpKamgrAwIED2bFjR7XbuP766wFISUmhX79+dOzo3EuQlJREXl4ebdu2rXK7U6ZM4Z133uGVV14hOzu7ym0sXbqUG2+8kdjY2HLbPHHiBF999RW33HJL6bKnT58u/frWW28lLCyM5ORkkpKSSo9WKkpPT6dZs2Y0a9aMDh06sHfvXhISEsqd8jt9+jTR0dGsXLmSv//979x7770sXbq02u+NqV67mHZM7DORiX0mlqv8+j9f/x+e++Y5Lu18KemJ6Vb51UT5NUhEZDQwAwgHMlT1+Qqv3w9MB4qBE8A0Vd0oIpFABjDAHePrqvqciPQC5nu9RRLwlKq+4Ms4qztyaCjNmzcv/frzzz/nX//6F8uXLyc2NpZhw4ZVegd+s2bNSr8ODw+v9tSW9/JhYWHl1g0LC6OoqKja7Z48eZL8/HzACYWWLVtWuZ3K7s0pKSkhLi6uyhCquE5V9/dU3OeioqKzlklISGD8+PEA3Hjjjdxzzz1VjtWcu6oqv77I/4KYiBiu7nI16UnpVvnVhPjtGomIhAMvAWOAvsDtItK3wmJ/U9UUVU0Ffgf80X3+FqCZqqYAA4F/E5HuqrpZVVPd5QcCJ4EF/toHf2rZsiXHjx+v9LWjR48SHx9PbGwsmzZt4uuvv26QMVW33ccff5xJkybx9NNPM3Xq1Crf48orr2TBggUUFBRw/PhxPvjgAwBatWpFYmIi77zzDuBUVn333Xel673zzjuUlJSwfft2cnJy6NWrV7Xfo+rccMMNpafMlixZQs+ePc/5PUzNqqr8+nL3l0xfPJ3hbw/nma+fYfXe1Vb51cj582L7YGCbquao6hlgHjDOewFV9b4w0Rzw3B2pQHMRiQBigDNAxYsYI4DtqrrTH4P3t7Zt23LZZZdxwQUX8B//8R/lXhs9ejRFRUVceOGF/PrXv+aSSy5pkDFVtd0lS5awYsWK0jCJiopizpw5lb7HgAEDuO2220hNTWX8+PFcccUVpa+9+eabzJo1i4suuoh+/frx/vvvl77Wq1cvrrrqKsaMGcMrr7xCdHQ0V199NRs3bix3sb0qaWlp7NnjXAB+4okneO+990hJSeHJJ58kIyPD12+NqYGn8us3Q3/D57d+zotXv8glHS/h/W3vc/dHdzPmvTG8sOoFth7eGuihGj/w253tInIzMFpVp7iP7wSGqOpDFZabDjwGRAHDVXWre2rrDZywiAUeVdVXK6w3G1itqn+uYvvTgGkAXbt2HbhzZ/m8aYp3NQeryZMnM3bsWG6++eYG3a79G/C/0p5fuZl8vcfp+ZUcn1za86tTi06BHqKpRm3vbPfnEUllJ7nPSi1VfUlVewCPA5461ME41006AYnAv4tIUukbi0QB1wPvVLVxVX1VVQep6qD27dvXfS+MMXXmqfx65ZpXWHzLYp4c/CSxEbHMWD2DUe+N4u4P72b+pvkcPnU40EM1PvDnxfZ8oIvX4wSguuLzecDL7tcTgY9UtRDYJyLLgEFAjvv6GJyjkb31O+TQN336dJYtW1buuUceeaTeLzgfPHiQESNGnPX84sWLz6ouq8ncuXPraVQmmLWNaVtp5dcz3zzD898+z6WdLyUtMY2ru1xtlV8hxp9BsgJIFpFEYDcwAScgSolIsqp6TpqmA56vdwHDReSvOKe2LgG8K7NuB97y49hD1ksvvdQg22nbtm21ZcDGVMe78mvL4S2lsz1a5Vdo8luQqGqRiDwEfIxT/jtbVTeIyNPASlVdCDwkItcAhcBh4G539ZeAOcB6nFNkc1R1LYCIxALXAv/mr7EbYxqGiNCrTS96tenFLwb+gtV7V5OZm1l682NcszhGdR9FWmIaqR1SCRNrxhGMrI28abLs30DwKiwuZNmeZWTmOLM9nio+RafmnZzZHpPS6BlvJd0NwdrIG2NCVmR4JMO6DGNYl2HlKr/mbpjLrPWzrPIryFiQGGOCmnfPr4MFB1m0cxGZOZnMWD2DGatn0L9Df9IT0xnZfaT1/AoQO+EYIDYfSc3qOh/JFVdcQWpqKqmpqXTq1IkbbrihTts3wadtTFtu7307f037K1k3ZfFw/4c5dvoYz3zzDMPfHs70xdPJzMnkZOHJQA+1SbEjkgCx+UhqVtf5SLwbNI4fP55x48ZVs7QJVV1admHqhVOZkjLFqfzKzSQrp6zya1iXYYxNGmuVXw3AjkgCxOYj8d98JB7Hjx/n008/tSOSRs5T+fXYQKfn15xRcxibNJav9nxlPb8aiB2RAHz4BPy4rn7f8/wUGPN8lS/bfCT+m4/EY8GCBYwYMYJWrVrV4gMzjUGYhDHo/EEMOn+QM9vjnmVk5WTx/rb3mb95Ph2bd2RM4hjSk9Kt8qseWZAECZuPpP7mI/F46623mDJlSrXfE9N4VVb5lZWbxWsbXmP2+tn8LO5npCelMyZxDJ1bdA70cEOaBQlUe+TQUGw+kvqbjwScFi7ffvttaSiapq2yyq+snKxylV9piWmM6j7KKr/qwK6RBIjNR+K/+Ug87zd27Fiio6PrtL5pvDyVX2+kvcGHN31YWvn17DfPMvzt4Tz4rwf5Z84/rfLrHFiQBIjNR+K/+UgA5s2bx+233+7Lt8M0AQktE5h64VQWjFvAu9e9y5397mTrka08ufRJhr09jP/84j9ZkreEwpLCQA81qFmLFBNwNh+JCSYlWsLqvavJys1i0c5FHD19lLhmcYzsNpK0pDT6d+jfZHp+WYsUY4ypg6oqvxZuX8jbW94urfxKS0yjV5tegR5uULAjkkYmFOcjCZTG+m/A+MfJwpMs3rWYrNwslu9ZTrEWN/rKr9oekViQmCbL/g2Yujp06hAf7/iYrJwssvc7lYieyq+R3UfSJrpNgEdYPyxIvFiQmMrYvwFTH/KP55fO9rj96HbCJZxLO11KWlIaw7sMD+nZHu0aiTHGNABP5Zd3z68Pcz/kyaVPlvb8Sk9M59JOlxIZ3jh7flmQGGNMPSg32+OAX7Bm3xoyczJZtHMRH+Z+SOtmrRnVbVSjrPyyIDHGmHoWJmEMPG8gA88byJODn+SrPV+RmZNZaeVXz/ieVXZyCBWNJxJDjM1HUrO6zkeyePFiBgwYQGpqKpdffjnbtm2r0/aNqQ+R4ZFc1eUqfnfV71hy2xKeu+I5esT14LUNr3HzBzdz08KbyFiXwe4TuwM91DqzIAmQqoKkuLi42vWysrKIi4vz17CCSsUgqa0HHniAN998k+zsbCZOnMgzzzzjh9EZc+5iI2MZmzSWl695mU9v/ZRfDvklLSJbMGP1DEa/N5o7s+5k3qZ5HDp1KNBDPSd2agv472//m02HKu82W1e92/Tm8cGPV/m693wkkZGRtGjRgo4dO5Kdnc3GjRu54YYbyMvL49SpUzzyyCNMmzYNcOYjWblyJSdOnGDMmDFcfvnlfPXVV3Tu3Jn333+fmJiYSrc3bNgw+vfvz6pVq9i/fz+vv/46zz33HOvWreO2224r/WFb2XZ37tzJNddcw/Lly2nTpg1XXXUVv/71r6tsJf/ss8/y+uuv06VLF9q3b1/aRn779u1Mnz6d/fv3Exsby8yZM+nduzeTJ08mOjqaDRs2sHfvXv74xz8ycuRInnrqKQoKCvjyyy958skngbL5SHbt2sUvfvELHn744bO2LyIcO3YMcPqHebeWNyZYtIluw4TeE5jQewL5x/P5aMdHZOZk8uw3z/L8t88ztNNQ0pPSQ6Lyy4IkQGw+Ev/NR5KRkUFaWhoxMTG0atWqwZpeGlNXCS0TmJIyhfsuuI8th7eQlZtFVm4WTy59kujwaK7ucjXpScFb+WVBAtUeOTQUm4+k/uYj+d///V+ysrIYMmQIv//973nsscfIyMio9ntjTDDwrvx6ZMAjrNm3hqycLD7e+TEf7nAqv0Z2G0l6UnpQVX5ZkAQJm4+kfuYj2b9/P9999x1DhgwB4LbbbmP06NFVjtWYYOVd+fXE4Cecyq/cTD7Y/gHvbHmH85uf78z2mJge8Mqv4IizJsjmI/HPfCTx8fEcPXqULVu2APDJJ5/Y3esm5JVWfl1ZVvmVHJfM6xteL638mrl2ZsAqv+yIJEC85yOJiYnhvPPOK31t9OjRvPLKK1x44YX06tWrQecjqWy7nvlIli1bRnh4OO+99x5z5syptBGk93wk3bp1O2s+kgceeIBnnnmGwsJCJkyYwEUXXQSUzUeyd+/ecvORPP/886SmppZebK+K9zWSmTNnMn78eMLCwoiPj2f27Nn1+F0yJrA8lV9jk8Zy6NQhFu1YRFZuFi+ueZEX17xIavtU0pKc2R4bqueX9doyAWfzkRjjO+/Kr21HthEu4QztNJRnL3+2zoFivbaMMaYJ8VR+TUmZwuZDm8nKzWLl3pW0jmrt921bkDQyoTgfydy5c+tpVMYYoLTyq6FYkDQyL730UoNsp23bttWWARtjmg6/Vm2JyGgR2Swi20TkiUpev19E1olItoh8KSJ93ecjReQ197XvReRJr3XiRORdEdnkvja0ruNrCteHTOXsszem/vgtSEQkHHgJGAP0BW73BIWXv6lqiqqmAr8D/ug+fwvQTFVTgIHAv4lId/e1GcBHqtobuAj4vi7ji46O5uDBg/YDpQlSVQ4ePEh0dHSgh2JMo+DPU1uDgW2qmgMgIvOAccBGzwKqesxr+eaA56e6As1FJAKIAc4Ax0SkFXAlMNld/4z72jlLSEggPz+f/fv312V1E+Kio6NJSEgI9DCMaRT8GSSdgTyvx/nAkIoLich04DEgChjuPv0uTuj8AMQCj6rqIRFJBfYDc0TkImAV8Iiq/nSug4uMjCzXksQYY0zd+PMaSWX36591HklVX1LVHsDjwK/cpwcDxUAnIBH4dxFJwgm+AcDLqtof+Ak469oLgIhME5GVIrLSjjqMMcZ//Bkk+UAXr8cJwJ5qlp8H3OB+PRHnOkihqu4DlgGD3PfMV9Vv3OXexQmWs6jqq6o6SFUHtW/f3ofdMMYYUx1/BskKIFlEEkUkCpgALPReQESSvR6mA1vdr3cBw8XRHLgE2KSqPwJ5IuIpkB6B1zUXY4wxDc+vLVJEJA14AQgHZqvqsyLyNLBSVReKyAzgGqAQOAw8pKobRKQFMAen2kuAOar6e/c9U4EMnGsqOcA9qnq4hnHsB3bWcTfaAQfquG6waSz70lj2A2xfglVj2Rdf96ObqtZ4SqdJ9NryhYisrE2vmVDQWPalsewH2L4Eq8ayLw21H9ZG3hhjjE8sSIwxxvjEgqRmrwZ6APWosexLY9kPsH0JVo1lXxpkP+waiTHGGJ/YEYkxxhifWJB4EZEdXt2IV7rPtRGRT0Rkq/t3fKDHWRtV7MtvRWS3+1y2W54d9Crr+ByKn0sV+xFyn4mI9PIab7aIHBORX4ToZ1LVvoTc5wIgIo+KyAYRWS8ib4lItHsv3zfu5zLfva+vfrdrp7bKiMgOYJCqHvB67nfAIVV93m2FH6+qjwdqjLVVxb78Fjihqn8I1LjqQkReA5aqaob7nyAW+H8Jsc+liv34BSH4mXi4Xb534/TRm06IfSbeKuzLPYTY5yIinYEvgb6qWiAibwNZQBrwd1WdJyKvAN+p6sv1uW07IqnZOOA19+vXKGvjYhqAV8fnWeB0fFbVI4TY51LNfoS6EcB2Vd1JiH0mlfDel1AVAcS4ndNjcRrfDsdpJwV++lwsSMpTYJGIrBKRae5z56nqDwDu3x0CNrpzU9m+ADwkImtFZHYonHoAkijr+LxGRDLctjmh9rlUtR8Qep+JtwnAW+7XofaZVOS9LxBin4uq7gb+gNNi6gfgKE6H9COqWuQulo/Tmb1eWZCUd5mqDsCZjGu6iFwZ6AH5oLJ9eRnoAaTi/EP7nwCOr7Zq3fE5yFW1H6H4mQDgnp67Hngn0GPxVSX7EnKfixt243A6pnfCmeNpTCWL1vv1DAsSL6q6x/17H7AAp539XhHpCOD+vS9wI6y9yvZFVfeqarGqlgAzcfYv2FXV8TnUPpdK9yNEPxOPMcBqVd3rPg61z8RbuX0J0c/lGiBXVferaiHwd+BSIM491QU1d2GvEwsSl4g0F5GWnq+BkcB6nI7Fd7uL3Q28H5gR1l5V++L5T+66EWf/glo1HZ9D6nOpaj9C8TPxcjvlTwWF1GdSQbl9CdHPZRdwiYjEiohQ9n/lM+Bmdxm/fC5WteUSZ+KsBe7DCJz55J8VkbbA20BXnA/qFlU9FKBh1ko1+/IGzqG6AjuAf/Oc0w5mUknHZ5xfgkLtc6lsP14kND+TWJwZUJNU9aj7XMj9X4Eq9yVU/6/8F3AbUASsAabgXBOZB7Rxn7tDVU/X63YtSIwxxvjCTm0ZY4zxiQWJMcYYn1iQGGOM8YkFiTHGGJ9YkBhjjPGJBYkxxhifWJAYEwAiMldEbq55SWOCnwWJMcYYn1iQGOMSke7uhFMz3cmBFolIjIh8LiKD3GXauXO9ICKTReQfIvKBiOSKyEMi8pjb3fdrEWlTy+0OFJElbqfmj736VU0VkRUi8p2IvOe2vmgtzqRlYe4ysSKSJyKRItJDRD5y32epiPR2l7nFnejoOxH5wi/fPNOkWZAYU14y8JKq9gOOAONrWP4CYCJOU79ngZNud9/lwF01bUxEIoE/ATer6kBgtvs+4ExGdLGqXgR8D9zntvD4DrjKXeY64GO3Sd+rwM/d9/l/gL+4yzwFjHLf5/qaxmTMuYqoeRFjmpRcVc12v14FdK9h+c9U9ThwXESOAh+4z68DLqzF9nrhhNEnTp89wnHalgNcICLPAHFAC+Bj9/n5OP2UPsOZQ+MvItICp9PrO+77ADRz/14GzHVnzPt7LcZkzDmxIDGmPO9mdsVADE4DPM/Re3Q1y5d4PS6hdv+/BNigqkMreW0ucIOqficik4Fh7vMLgefcU2cDgU9x5p44oqqpFd9EVe8XkSFAOpAtIqmqerAWYzOmVuzUljE124HzAxvK2nHXl81AexEZCs6pLhHp577WEvjBPf01ybOCqp4AvgVmAP905804BuSKyC3u+4iIXOR+3UNVv1HVp4ADQJd63gfTxFmQGFOzPwAPiMhXQLv6fGNVPYMTTv8tIt8B2TinqAB+DXwDfAJsqrDqfOAO92+PScB97vtswJktD+D3IrJORNYDX+BcYzGm3lgbeWOMMT6xIxJjjDE+sYvtxviRiLwEXFbh6RmqOicQ4zHGH+zUljHGGJ/YqS1jjDE+sSAxxhjjEwsSY4wxPrEgMcYY4xMLEmOMMT75/wF2OSUszrjWHQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2a6b6ee8c18>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(num_leaves))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(num_leaves))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "#    pyplot.plot(num_leaves, -test_scores[i], label= 'valid_max_depth:'   + str(value))\n",
    "    pyplot.plot(num_leaves, -train_scores[i], label= 'train_max_depth:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'num_leaves' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3、调整叶子节点数目正则系数和叶子节点分数正则系数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0.4, 0.6, 0.8], 'reg_lambda': [0.2, 0.25, 0.3]}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha=[0.4,0.6,0.8]\n",
    "reg_lambda=[0.2,0.25,0.3]\n",
    "param_2=dict(reg_alpha=reg_alpha,reg_lambda=reg_lambda)\n",
    "param_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=0.8,\n",
       "        learning_rate=0.1, max_bin=256, max_depth=8, min_child_samples=20,\n",
       "        min_child_weight=1, min_split_gain=0.0, n_estimators=363,\n",
       "        n_jobs=-1, nthread=6, num_leaves=50, objective='binary',\n",
       "        random_state=None, reg_alpha=0, reg_lambda=0, scale_pos_weight=1,\n",
       "        seed=0, silent=True, subsample=0.8, subsample_for_bin=200000,\n",
       "        subsample_freq=1),\n",
       "       fit_params=None, iid=True, n_jobs=6,\n",
       "       param_grid={'reg_alpha': [0.4, 0.6, 0.8], 'reg_lambda': [0.2, 0.25, 0.3]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=3)\n",
    "\n",
    "params = {'boosting_type': 'gbdt', \n",
    "          'objective': 'binary', \n",
    "          'nthread': 6, \n",
    "          'silent': True,\n",
    "          'learning_rate': 0.1, \n",
    "          'num_leaves': 50, \n",
    "          'max_depth': 8,\n",
    "          'max_bin': 256, \n",
    "          'subsample_for_bin': 200000,#默认200000，用来构建直方图数据数量\n",
    "          'subsample': 0.8, #默认1\n",
    "          'subsample_freq': 1, \n",
    "          'colsample_bytree': 0.8, \n",
    "          'reg_alpha': 0,          \n",
    "          'reg_lambda': 0,  \n",
    "          'min_split_gain': 0.0, \n",
    "          'min_child_weight': 1,         \n",
    "          'min_child_samples': 20, \n",
    "          'scale_pos_weight': 1,\n",
    "         }\n",
    "lgbm_2=lgbm.sklearn.LGBMClassifier(n_estimators=363,seed=0,**params)\n",
    "gsearch2_2=GridSearchCV(lgbm_2,param_grid=param_2,scoring='neg_log_loss',n_jobs=6, cv=kfold)\n",
    "gsearch2_2.fit(X_train,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mean_fit_time': array([199.43567896, 230.59458598, 250.3221283 , 280.27625195,\n",
      "       275.80514717, 282.68814778, 277.69193602, 280.82961917,\n",
      "       175.63436182]), 'std_fit_time': array([ 9.87744887, 15.30693881,  7.08267799, 29.03009709,  8.49832812,\n",
      "       22.29968038,  7.62978159, 16.3946438 , 37.76137706]), 'mean_score_time': array([22.02594384, 37.66689809, 17.88282228, 17.94125414, 17.45678576,\n",
      "       18.41700625, 17.29101022, 18.44040386, 15.82778406]), 'std_score_time': array([5.78018803, 5.05956246, 0.40386711, 5.37539628, 1.45118404,\n",
      "       6.34880233, 0.73238113, 7.0913777 , 4.08753164]), 'param_reg_alpha': masked_array(data=[0.4, 0.4, 0.4, 0.6, 0.6, 0.6, 0.8, 0.8, 0.8],\n",
      "             mask=[False, False, False, False, False, False, False, False,\n",
      "                   False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_reg_lambda': masked_array(data=[0.2, 0.25, 0.3, 0.2, 0.25, 0.3, 0.2, 0.25, 0.3],\n",
      "             mask=[False, False, False, False, False, False, False, False,\n",
      "                   False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'reg_alpha': 0.4, 'reg_lambda': 0.2}, {'reg_alpha': 0.4, 'reg_lambda': 0.25}, {'reg_alpha': 0.4, 'reg_lambda': 0.3}, {'reg_alpha': 0.6, 'reg_lambda': 0.2}, {'reg_alpha': 0.6, 'reg_lambda': 0.25}, {'reg_alpha': 0.6, 'reg_lambda': 0.3}, {'reg_alpha': 0.8, 'reg_lambda': 0.2}, {'reg_alpha': 0.8, 'reg_lambda': 0.25}, {'reg_alpha': 0.8, 'reg_lambda': 0.3}], 'split0_test_score': array([-0.39984656, -0.39978411, -0.39986698, -0.39981514, -0.399825  ,\n",
      "       -0.39979179, -0.39973647, -0.39969969, -0.39970353]), 'split1_test_score': array([-0.39989182, -0.39986568, -0.39991809, -0.39981704, -0.39985204,\n",
      "       -0.3998739 , -0.3998554 , -0.39985465, -0.39986746]), 'split2_test_score': array([-0.39968158, -0.39974506, -0.39969229, -0.39970101, -0.39976896,\n",
      "       -0.39974087, -0.39969531, -0.39966234, -0.39970319]), 'mean_test_score': array([-0.39980665, -0.39979828, -0.39982579, -0.39977773, -0.39981534,\n",
      "       -0.39980219, -0.39976239, -0.39973889, -0.39975806]), 'std_test_score': array([9.03499856e-05, 5.02542877e-05, 9.66737295e-05, 5.42536719e-05,\n",
      "       3.45990315e-05, 5.48042453e-05, 6.78780831e-05, 8.32585566e-05,\n",
      "       7.73591385e-05]), 'rank_test_score': array([7, 5, 9, 4, 8, 6, 3, 1, 2]), 'split0_train_score': array([-0.39759313, -0.39752962, -0.39761232, -0.39756882, -0.39758687,\n",
      "       -0.39754199, -0.39751379, -0.3975134 , -0.39751694]), 'split1_train_score': array([-0.39747535, -0.3974704 , -0.3975111 , -0.39744537, -0.39747674,\n",
      "       -0.3975189 , -0.39746547, -0.3974886 , -0.39748451]), 'split2_train_score': array([-0.39749971, -0.39757013, -0.39748671, -0.39751549, -0.39755614,\n",
      "       -0.39753104, -0.39753309, -0.39753379, -0.39754296]), 'mean_train_score': array([-0.39752273, -0.39752338, -0.39753671, -0.39750989, -0.39753992,\n",
      "       -0.39753064, -0.39750412, -0.39751193, -0.3975148 ]), 'std_train_score': array([5.07649422e-05, 4.09544637e-05, 5.43850248e-05, 5.05543382e-05,\n",
      "       4.63983087e-05, 9.42845255e-06, 2.84421157e-05, 1.84780309e-05,\n",
      "       2.39096863e-05])}\n",
      "-0.3997388919872223\n",
      "logloss of train : 0.398003298291121\n",
      "logloss of test : 0.40286772938280563\n"
     ]
    }
   ],
   "source": [
    "print(gsearch2_2.cv_results_)\n",
    "print(gsearch2_2.best_score_)\n",
    "y_pred_train = gsearch2_2.predict_proba(X_train)\n",
    "logloss = log_loss(y_train, y_pred_train)\n",
    "print (\"logloss of train :\",logloss )\n",
    "\n",
    "y_pred_test = gsearch2_2.predict_proba(X_test)\n",
    "logloss = log_loss(y_test, y_pred_test)\n",
    "print (\"logloss of test :\",logloss )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.399772 using {'colsample_bytree': 0.8, 'subsample': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEKCAYAAAAvlUMdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8leWd///XJyd7SFgCiYGAgICAgmxiUccBlRZtq13ma9XqqF38qTi12s3u1m87P6etHdfOVOvSZSq1Yx11xh1rrYqDYXFhk32XHcIWyPL5/nHf5+QkJOGQ5M5JyPs5cx6cc5/73Oe6hOadz31f93WZuyMiItLeMtLdABEROT4pYEREJBIKGBERiYQCRkREIqGAERGRSChgREQkEgoYERGJhAJGREQioYAREZFIZKa7AenUt29fHzx4cLqbISLSpcybN2+7u/c72n7dOmAGDx5MRUVFupshItKlmNnaVPbTKTIREYmEAkZERCKhgBERkUgoYEREJBIKGBERiYQCRkREIqGAERGRSHTr+2Baa97aXby5YjujyooYWVbIgF55mFm6myUi0qkoYFqhYs1O7nzpg8TrotxMRpYVMeqEQkaVFTGqrIgRpYXkZcfS2EoRkfQyd093G9Jm0qRJ3to7+fcdqmHZh5Us3ryXpZsrWbK5kqUf7uXA4VoAMgwG9y0IAicpeMp65qraEZEuzczmufuko+2nCqaVeuRkMvHEPkw8sU9iW12ds37XAZZsrmTJ5r0s2VzJuxt28z/vbk7s0zMvi5GJwClMVDu5Wap2ROT4ooBpRxkZxonFBZxYXMCMU8sS2/dWVbPswyBwloR/Pl6xvkG1MyRe7SQFzwlFqnZEpOtSwHSAwtwsJg3uw6TBDauddTvj1U4QPAvX7+a/k6qdXvlJ1c4JQfgML+2hakdEugQFTJpkZBiD+xYwuG8BF4ypr3Yqk6ud8FTbrLnrOVgdVDuxDEuqdgoTwVNalKNqR0Q6FQVMJ1OUm8Xpg/twelK1U1vnrN2xn6VJwTN/7S6eeWdTYp/e+VmMPKHhKbZhJap2RCR9FDBdQCzDGNqvB0P79eDCpGpnz8Fqloaj1+LB84e5a6mqrkt87qR+BUcET0mhqh0RiZ4CpgvrmZfFGUOLOWNocWJbbZ2zZsf+YNh0OJKtYs1Onk6qdvoUZDOqrLBB8Awr6UFOpqodEWk/kQaMmc0A7gZiwK/d/Y5G718HzARqgX3Ate6+2MyygV8Bk4A64CZ3fzX8zGXAdwAHNgFXuPv2pGN+HfgZ0C95e3cRVC09OKlfDz4xtn77ngPVLPmwsj54Pqzk92+t5VBNUO1khp8bVVYY3DQaBk+/Hqp2RKR1IrvR0sxiwAfAdGAD8DZwmbsvTtqnyN0rw+cXATe4+wwzmwlMcvdrzKwEeA44nWDutE3AaHffbmY/BQ64+23hMQYCvwZGAhOPFjBtudHyeFBb56zeHlY7H9bfu7N5T1Vin+KC7ETYxCueYSU9yM7UNHYi3VVnuNFyMrDC3VeFDZoFXAwkAiYeLqECgqoEYDQwO9xnq5ntJqhmFgAGFJjZDqAIWJF0jH8Fvgk8FUWHjjexDGNYSQ+GlfTgk6f1T2zffeBwImziwfObOWs5nFTtDCvpcUTw9CvMSVdXRKQTijJgBgDrk15vAM5ovFNYrdwCZAPnhpvfAS4OQ2kgMBEY6O5zzex64D1gP7Cc4BRbvALa6O7v6JRO2/TKz2bKScVMOan+2k5NbR1rduxncTx4NlcyZ+UOnlywMbFP3x45iYEE8eA5qZ+qHZHuKsqAaeqn/BHn49z9fuB+M7sc+B5wFfAwMAqoANYCbwI1ZpYFXA+MB1YB9wLfNrNfAN8FPnrURpldC1wLMGjQoGPvVTeVGctgWEkhw0oKuSip2tm5/3CD02tLP6zk0TfXJKqdrFhwbWd04rpOMAN13x6qdkSOd1Feg5kC3ObuHwtffxvA3f//ZvbPAHa5e88m3nsT+BLBabQ73P28cPs5wK3AtwhOqR0IP1JOcK1msrt/2Fwbu/s1mKjU1NaxKry2kxw8WyoPJfbpV5jTYCLQkWWFnNSvB1kxVTsinV1nuAbzNjDczIYAG4FLgcuTdzCz4e6+PHz5cYJTXphZPkH47Tez6UBNOLqsPzDazPq5+zaCAQRL3P09oCTpuGsIBgl0u1FknUFmLIMRpYWMKC3k4nH123fuP9xghoIlmyt5ZOUODtcG1U52LINhJT0YWVaYqHhGnlBIsaodkS4psoBx9xozuxF4gWCY8sPuvsjMbgcq3P1p4EYzOx+oBnYRnB6DICxeMLM6gnC6MjzmJjP7EfCamVUTnD67Oqo+SPvqU5DNWcP6ctawvolt1bV1rNoWVjvhqbbXl2/nz/Prr+2UhNVOcvAM6Vugakekk9N6MDpF1ilt33cocaNoPHhWbN1LdW3w7zU7lsHw0h6JKicePL0LstPccpHjX6qnyBQwCpgu43BNHau270vcLLo4PNW2fV/9tZ3Sopz6wQRh8AzpW0Cmqh2RdtMZrsGItKvszAxGnlDEyBOKgnGEoW17D4Uj2eqD540V2+urncwMRpT2SMw8HT/V1itf1Y5IlBQw0uX1K8yhX2E//m54v8S2wzV1rNy2LzGoYOmHe/nLsq38ad6GxD5lPXOTVhcN7t0ZXKxqR6S9KGDkuJSdmZEIjmRb91bVX9sJg+dvy7dTUxdUOzmZGZx8QmEieEaeUMTosiJ65meloxsiXZoCRrqVksJcSgpzOWdEfbVzqKaWFVv3NRhU8PKSrTxeUV/t9O+ZG04CWl/xDC4uIJahWSNEmqOAkW4vJzPGKf17ckr/+nt83Z1tew+xuNF6O3/9YBu1YbWTm5XByaXxudjiN4wW0TNP1Y4IKGBEmmRmlBTlUlKUy9STE/fwcqimluVb9iVOry3ZXMmLiz/kjxX10+4N6JV3xHo7J6rakW5IASNyDHIyY5w6oCenDmhY7WwNq53khd7+sqy+2snLijHihEJGJwXPyLJCinJV7cjxSwEj0kZmRmlRLqVFuUxLqnaqqoNrO8nB89z7H/LY3MbVTlEQPOG1nRP75JOhakeOAwoYkYjkZjVd7XxYWZW4Xyd+mu2VpVsIix3ysmIML+1B7/xseuRmUpiTSY+cTHrkBn8W5mbSIycreC/+fvheQXamwkk6DQWMSAcyM8p65lHWM49pIxtWO/FrO4s3V7Jy2z52HzjM+l0H2FdVw75DNRw4XJvSd/RoIpAKw+eJYEp6P/66MDerfltOpq4ZSZspYEQ6gdysGGPKezKm/IjVKhJqauvYf7iWfYdqwtCpprKqJhFA+6pq2Jv03r5DNeytCh6b91TV73eoJqU25WfHGgRQfWBlJVVSTVRWSaFWkJOpSUm7MQWMSBeRGcugZ15Gm4dB19U5+w8fGUp7w2DamxRY+w4lh1YN2/ceYG9VdbDtUA2pTGWYm5VBj5yspCoquWqKB1RWC6cDg1DTyqhdjwJGpJvJyDAKc7MozM2C5gumo3J3DoQVVcNQalRZNXg/qKzW7zzQYHt8tF1LsjMzGp7aO6KKymp0OjAeZFkNAisnMwMtq94xFDAi0ipmRkFOcBqstOjo+zfH3amqrmPvoeomT/ftDUMpuZKKv79pd1VSiFUnJjhtSVbMGoZSE9ejEuHVxOnA+P55WTEF1VEoYEQkrcyMvOwYedkxSgpbfxx351BNXcPTe4kqKgivysan/8KKa+veKlZtq992qKbuqN+XYSRO3zV9Parhqb/604ENRwLmZ8WO25F/ChgROS6YGblZMXKzYvRt4zLbh2pq2X+oNqyUqps53VdTfz0q3LZz/2HW7TiQ2Haw+ugj/8ygR3aj8MnNavZ6VPJgi+RtBdmdb+SfAkZEpJGczBg5mTH6tHGF1JraukaVVBOj/cLXe5OuW+05WM3GXQcS++9PcYh6QXYs6XRfFkVNDKyInxqcPKQPw0p6tKl/R6OAERGJSGYsg1752W1e3K42HPlXH0JNjPZrcH2q/v0P91TV73e4fuTfP396jAJGRKS7i2UYRblZbZ67rq7OOVAdnPoryIm1U+uap4AREekmMjIsccqsQ76vQ75FRES6HQWMiIhEQgEjIiKRUMCIiEgkFDAiIhIJBYyIiERCASMiIpFQwIiISCQUMCIiEgkFjIiIREIBIyIikVDAiIhIJBQwIiISiUgDxsxmmNkyM1thZrc28f51ZvaemS00s9fNbHS4PdvMHgnfe8fMpiZ95rJw+7tm9ryZ9Q23/8zMlobbnzSzXlH2TUREWhZZwJhZDLgfuAAYDVwWD5Akf3D3Me4+Dvgp8Itw+5cB3H0MMB2408wyzCwTuBuY5u5jgXeBG8PPvAScGm7/APh2VH0TEZGji7KCmQyscPdV7n4YmAVcnLyDu1cmvSwAwrXWGA3MDvfZCuwGJgEWPgrMzIAiYFO434vuXhN+/i2gPIpOiYhIaqIMmAHA+qTXG8JtDZjZTDNbSVDBfCXc/A5wsZllmtkQYCIw0N2rgeuB9wiCZTTwUBPf/QXgufbqiIiIHLsoA8aa2OZHbHC/391PAr4FfC/c/DBBIFUAdwFvAjVmlkUQMOOB/gSnyBqcCjOz7wI1wH802Siza82swswqtm3b1pp+iYhICqIMmA3AwKTX5YSns5oxC/gUgLvXuPvN7j7O3S8GegHLgXHh+yvd3YHHgTPjBzCzq4BPAJ8P3z+Cuz/g7pPcfVK/fv1a3zsREWlRlAHzNjDczIaYWTZwKfB08g5mNjzp5ccJQgQzyzezgvD5dKDG3RcDG4HRZhZPhunAknC/GQRV0EXufiC6bomISCoyozqwu9eY2Y3AC0AMeNjdF5nZ7UCFuz8N3Ghm5wPVwC7gqvDjJcALZlZHECpXhsfcZGY/Al4zs2pgLXB1+Jn7gBzgpeD6P2+5+3VR9U9ERFpmzZxJ6hYmTZrkFRUV6W6GiEiXYmbz3H3S0fbTnfwiIhIJBYyIiERCASMiIpFQwIiISCQUMCIiEgkFjIiIREIBIyIikVDAiIhIJBQwIiISCQWMiIhEQgEjIiKRUMCIiEgkFDAiIhKJyKbrF5Gup7q6mg0bNlBVVZXupkgnkJubS3l5OVlZWa36vAJGRBI2bNhAYWEhgwcPJlxXSbopd2fHjh1s2LCBIUOGtOoYOkUmIglVVVUUFxcrXAQzo7i4uE3VrAJGRBpQuEhcW/8tKGBERCQSRw0YMzvJzHLC51PN7Ctm1iv6polId7N7925++ctfHvPnLrzwQnbv3h1Bi9rX4MGD2b59e5v3acnzzz/PySefzLBhw7jjjjua3GfdunVMmzaN8ePHM3bsWJ599tlWf19LUqlgngBqzWwY8BAwBPhDJK0RkW6tuYCpra1t8XPPPvssvXq17ffempqaNn2+M6itrWXmzJk899xzLF68mMcee4zFixcfsd+Pf/xjLrnkEhYsWMCsWbO44YYbImlPKqPI6ty9xsw+Ddzl7vea2YJIWiMincaPnlnE4k2V7XrM0f2L+OEnT2n2/VtvvZWVK1cybtw4srKy6NGjB2VlZSxcuJDFixfzqU99ivXr11NVVcVNN93EtddeCwS/9VdUVLBv3z4uuOACzj77bN58800GDBjAU089RV5eXpPfN3XqVM4880zeeOMNLrroIv7xH/+R6667jnXr1gFw1113cdZZZ7Ft2zYuv/xyduzYwemnn87zzz/PvHnz6Nu3b5PHba6dcWvWrGHGjBmcccYZLFiwgBEjRvDb3/6W/Px8AO69916eeeYZqqur+dOf/sTIkSOZO3cuX/3qVzl48CB5eXk88sgjnHzyyQ2OO3fuXIYNG8bQoUMBuPTSS3nqqacYPXp0g/3MjMrK4O92z5499O/fv9m/k7ZIpYKpNrPLgKuA/w63tW5QtIhIC+644w5OOukkFi5cyM9+9jPmzp3LT37yk8Rv4Q8//DDz5s2joqKCe+65hx07dhxxjOXLlzNz5kwWLVpEr169eOKJJ1r8zt27d/PXv/6Vr33ta9x0003cfPPNvP322zzxxBN86UtfAuBHP/oR5557LvPnz+fTn/50IoCak0o7ly1bxrXXXsu7775LUVFRg8qtb9++zJ8/n+uvv56f//znAIwcOZLXXnuNBQsWcPvtt/Od73wHgE2bNnHhhRcCsHHjRgYOHJg4Tnl5ORs3bjziu2+77TZ+//vfU15ezoUXXsi9997bYn9aK5UK5hrgOuAn7r7azIYAv4+kNSLSabRUaXSUyZMnN7gH45577uHJJ58EYP369Sxfvpzi4uIGnxkyZAjjxo0DYOLEiaxZs6bF7/jc5z6XeP7yyy83OKVUWVnJ3r17ef311xPfO2PGDHr37t3iMVNp58CBAznrrLMAuOKKK7jnnnv4+te/DsBnPvOZRPv//Oc/A0GlcdVVV7F8+XLMjOrqagD69++fuIbi7ke0pamRYI899hhXX301X/va15gzZw5XXnkl77//PhkZ7Tvu66gB4+6Lga+EDe0NFLp701eORETaUUFBQeL5q6++yssvv8ycOXPIz89n6tSpTd6jkZOTk3gei8U4ePBgyt9RV1fHnDlzjjil1tQP7uak2s7GP/iTX8f7EIvFEteGvv/97zNt2jSefPJJ1qxZw9SpU484Znl5OevXr0+83rBhQ5Onvx566CGef/55AKZMmUJVVRXbt2+npKQk5X6mIpVRZK+aWZGZ9QHeAR4xs1+0aytERIDCwkL27t3b5Ht79uyhd+/e5Ofns3TpUt566612//6PfvSj3HfffYnXCxcuBODss8/m8ccfB+DFF19k165dzR4j1XauW7eOOXPmAEFFcfbZZ7fYtj179jBgwAAAHn300Sb3Of3001m+fDmrV6/m8OHDzJo1i4suuuiI/QYNGsTs2bMBWLJkCVVVVfTr16/F72+NVOqhnu5eCXwGeMTdJwLnt3tLRKTbKy4u5qyzzuLUU0/lG9/4RoP3ZsyYQU1NDWPHjuX73/8+H/nIR9r9+++55x4qKioYO3Yso0eP5t///d8B+OEPf8iLL77IhAkTeO655ygrK6OwsLDJY6TazlGjRvGb3/yGsWPHsnPnTq6//voW2/bNb36Tb3/725x11lkNRtUlX4PJzMzkvvvu42Mf+xijRo3ikksu4ZRTglOdP/jBD3j66acBuPPOO3nwwQc57bTTuOyyy3j00UcjucHWjlb6mdl7wEeB3wDfdfe3zexddx/b7q3pYJMmTfKKiop0N0Ok01iyZAmjRo1KdzM6nUOHDhGLxcjMzGTOnDlcf/31ieqmNdasWcMnPvEJ3n///XZsZTSa+jdhZvPcfdLRPpvKRf7bgReAN8JwGQosb1VLRUS6oHXr1nHJJZdQV1dHdnY2Dz74YLqb1CWkcpH/T8Cfkl6vAj4bZaNERNrTzJkzeeONNxpsu+mmm7jmmmtS+vzw4cNZsKDh7X87duzgvPPOO2Lf2bNnHzFirLHBgwd3ieqlrY4aMGZWDtwLnAU48Dpwk7tviLhtIiLt4v7772/3YxYXF7fpNFl3kMpF/keAp4H+wADgmXCbiIhIs1IJmH7u/oi714SPR4H2H88mIiLHlVQCZruZXWFmsfBxBXDkvAciIiJJUgmYLwCXAB8Cm4F/IJg+RkREpFlHDRh3X+fuF7l7P3cvcfdPEdx0eVRmNsPMlpnZCjO7tYn3rzOz98xsoZm9bmajw+3ZZvZI+N47ZjY16TOXhdvfNbPnzaxvuL2Pmb1kZsvDP1ueLEhEOh2tB9P29WC+8IUvUFJSwqmnntrqY7SX1s5sdsvRdjCzGHA/cAEwGrgsHiBJ/uDuY9x9HPBTID4FzZcB3H0MMB2408wyzCwTuBuYFt7o+S5wY/iZW4HZ7j4cmB2+FpEuROvBtN3VV1+dmGcs3VK50bIpqcwpMBlYEd43g5nNAi4GElOVhlPQxBUQDIOGIJBmh/tsNbPdwCRgQfjdBWa2AygCVoSfuRiYGj7/DfAq8K1j7JeIxD13K3z4Xvse84QxcEHzc+VqPZi2rQcDcM455xx1BumO0toKJpWpRQcA65Nebwi3NWBmM81sJUEF85Vw8zvAxWaWGS4PMBEY6O7VwPXAe8AmgiB6KPxMqbtvBgj/bHJaUDO71swqzKxi27ZtKXRDRDqK1oMJtHY9mM6m2QrGzPbSdJAY0PSvA0fu19gRx3P3+4H7zexy4HsEC5s9DIwCKoC1wJtAjZllEQTMeGAVwQ2g3wZ+nEJ74t/3APAABHORpfo5kW6nhUqjo2g9mGNbD6azaTZg3L3pqUJTtwEYmPS6nKDqaM4s4N/C764Bbo6/YWZvEsx/Ni58f2W4/XHqr7VsMbMyd99sZmXA1ja2X0TSTOvBHNt6MJ1N+y5f1tDbwHAzG2Jm2cClBDMCJJjZ8KSXHyecRNPM8s2sIHw+HagJFz7bCIw2s/iNntOBJeHzpwmqH8I/n2r/LolIlLQeTPNSWQ+ms4ksYMIq5EaCmZiXAI+7+yIzu93M4ivg3Ghmi8xsIcHItHhAlADzzWwJwYX6K8NjbgJ+BLxmZu8SVDT/HH7mDmC6mS0nCJ701/cicky0HkzzUlkPBuCyyy5jypQpLFu2jPLych566KGmDtchjroezPFM68GINKT1YJqm9WCiWw9GRKRb03owrZPKdP1NjSbbQzDC62vx+1xERDorrQeTHqlUML8gGP31B4Khx5cCJwDLCIYTT42qcZ3WO3+Et34Jw86H4dNhwCSIqRgU6ay0Hkx6pPJTcYa7n5H0+gEze8vdbzez70TVsE4tuwAyc+H1X8Dffg65PWHotCBshp0PhSeku4UiImmXSsDUmdklwH+Gr/8h6b3uOUJg1CeCx8FdsPIvsGI2rHgZFv9X8H7pGBh+PgybDgMnQywrve0VEUmDVALm8wQTTMbnMZgDXGFmedRPNNk95fWGUz8TPNxhy/uw/KUgbN68F17/V8gpgqF/H1Q2w6ZDzyNmyxEROS4dNWDCi/ifbObt19u3OV2YWTCR3wlj4O9ugao9sOqvsOKloMJZ8kywX8loGHZeEDaDpkBmdnrbLSISkaPeaGlm5Wb2pJltNbMtZvaEmZV3ROO6tNyeMPoiuOheuHkRXD8Hpv9fKOgLb/07/PYi+JfB8Nhl8PZDsLvlyfNEugOtB9O29WDWr1/PtGnTGDVqFKeccgp33313q47TXlI5RfYIwQiy/xO+viLcNj2qRh13zKB0dPA46ytwaC+s/ltQ3Sx/GZaFE9X1HRFUNsPOgxPPgqzc9LZbpIPFA+aGG25osL22tpZYLNbs59pjsseamhoyM7v2aNDMzEzuvPNOJkyYwN69e5k4cSLTp09n9OjGS3F1UHtS2Kefuz+S9PpRM/tqVA3qFnIKYeSFwcMdti8PT6W9DG//Gt66H7LyYfDfhUOhz4c+Q9Pdaulm/mXuv7B059J2PebIPiP51uTml2nSejBtWw+mrKyMsrIyIJjXbdSoUWzcuDFtAZPKXGTbzewKM4uFjyuAIxc3kNYxg34jYMpMuPJJ+NZquPxxGH8FbP8AnvsG3DM+eDz7zWAQweED6W61SCS0HkygPdaDWbNmDQsWLOCMM8444r2OkkoF8wXgPuBfCYYlvwmkdvurHLvsAhjxseABsGNlUNksfwnm/wbm/iq4B+fEs+rvuykeFgSVSDtqqdLoKFoPpnXrwezbt4/Pfvaz3HXXXRQVFbXY1iilMopsHXBR8rbwFNldUTVKkhSfFDzO+P+g+iCsfSO4brPiJXg+XAqn14n1YTPknCCkRI4DWg/m2NeDqa6u5rOf/Syf//znE0GVLq2drv+Wdm2FpCYrLwiRC+6Af5oHX1kIF/48GPq88A/w2KXByLTfXhzch7N1aXCNR6SL0HowzUtlPRh354tf/CKjRo3illvS/2O6tQGj8zGdQZ8hMPnLcPks+NYauPK/YPK1sPdDePF78Msz4K4x8MxNsOS/oaoy3S0WaZHWg2leKuvBvPHGG/zud7/jlVdeYdy4cYwbNy6tyym3aj0YM1vn7oMiaE+HOq7Xg9m9rn4Km1WvwuF9kJEZ3Nw57PzgUXqKrt1IA1oPpmlaD6ad14NpZpp+CKqXpsf8SefRaxBMuiZ41ByG9f9bf9/Nyz8MHoX9w1kFzoehUyGvV7pbLdIpaT2Y1tGKlsdrBdOSyk1BZbPiZVj5KhzaAxaDgWcEgTN8ejBhZ0ZkK2pLJ3W8VjBtXQ+mKW1ZD6YraUsFo4DpjgGTrLYaNrxdPxT6w3eD7QUl9Td5Dp0G+X3S207pEEuWLGHkyJFHjHCS7sndWbp0qZZMllaKZcGJZwaP834Ae7fAytlB2Cx7Ft75A1hGsKja8HAam7Lxqm6OU7m5uezYsYPi4mKFTDfn7uzYsYPc3NZPWaUKprtXMC2pq4WN8+qXINi0AHDI71t/7eak86Dg+Dkd0N1VV1ezYcOGJu/bkO4nNzeX8vJysrIarmmlU2QpUMAco/3b60emrZwNB3YABv3Hh9XNdBgwATKan5RQRLo+BUwKFDBtUFcLmxfWzyqwoQLwYBG2k86tnxW6R0m6Wyoi7UwBkwIFTDs6sBNWvlI/Om3/tmB72Wn1q3mWnw4xXfYT6eoUMClQwESkri4YjRZfzXP9XPBayOkJJ00Nq5vzoags3S0VkVbQKDJJn4wM6D8ueJzzDTi4O5hNIB44i58K9is9NRwKPT24ByeW1eJhRaRrUQWjCqZjucOWRfWzCqx/C+pqILsQhv59feD01KrcIp2VKhjpnMzghFODx9k3BxNwrv5r/VDopf8d7NdvVP2sAoOmQGZOy8cVkU5HFYwqmM7DHbYtrZ9VYO2bUFcNWQXBOjfDw8ECvU9Md0tFujVVMNL1mEHJqOBx5j/BoX2w5m9hdfMSfPBcsF/x8PpZBU48G7Jaf6exiERHFYwqmK7BHXasqA+bNW9A7SHIzIMhf1e/BEHxSeluqchxTxWMHF/MoO/w4DHlBjh8ANa8Ht538xIsfzHYr8/Q+vtuBp8N2fnpbbdIN6aAka4pOx9GfDR4AOxYGU5j8xLM/x3MfQBiOTD4rCBshk+H4mFaYE2kA+kUmU6RHX+qq2DtG/WDBXaVkLgsAAAUS0lEQVQsD7b3GlR/k+eQcyCnR3rbKdJFdYo7+c1sBnA3EAN+7e53NHr/OmAmUAvsA65198Vmlg38CpgE1AE3ufurZlYI/C3pEOXA7939q2Y2CPgN0Cv8vlvdvcXFqBUw3cSuNeGptNmw6q9QvR9i2fXLRw+fDv1GqroRSVHaA8bMYsAHwHRgA/A2cJm7L07ap8jdK8PnFwE3uPsMM5sJTHL3a8ysBHgOON3d6xp9xzzgZnd/zcweABa4+7+Z2WjgWXcf3FIbFTDdUM0hWDcnrG5ehm1Lgu1F5fX33Qz5e8gtSm87RTqxznCRfzKwwt1XhQ2aBVwMJAImHi6hAiCedqOB2eE+W81sN0E1Mze+s5kNB0qor2gciP9U6Alsauf+yPEgMweGTg0eH/0x7NlQfyrt/T/D/N9ARiYM/Eh43835wZQ2qm5EjlmUATMAWJ/0egNwRuOdwmrlFiAbODfc/A5wcRhKA4GJ4Z9zkz56GfBHry/BbgNeNLN/Igir85tqlJldC1wLMGjQoNb0S44nPcth4tXBo7Ya1v9vOBR6Nrx8W/AoLKtfYG3oNMjrld42i3QRUZ4i+z/Ax9z9S+HrK4HJ7v5Pzex/ebj/VWaWCfwMmAasBbKAX7n7U0n7LwaudPd54etbwv7caWZTgIeAUxufVkumU2TSosrN9csPrPwLHNoDFguWHYjPKnDCWC0fLd1OZzhFtoGg6ogrp+XTVrOAfwNw9xrg5vgbZvYmsDzp9WlAZjxcQl8EZoSfn2NmuUBfYGvbuiHdVlEZTLgyeNTWwIa36++7eeXHwaOgJGn56HMhv0+6Wy3SaUQZMG8Dw81sCLARuBS4PHkHMxvu7vHg+DhhiJhZPkE1st/MpgM1yYMDCE6PPdbo+9YB5wGPmtkoIBfY1s59ku4qlgknTgke530f9m2tv+/mg+fhncfAMmDAxPqh0P3Hq7qRbi3qYcoXAncRDBt+2N1/Yma3AxXu/rSZ3U1wraQa2AXc6O6LzGww8ALBEOWNwBfdfW3ScVcBF7r70qRto4EHgR4EF/y/6e4vttQ+nSKTdlFXCxvnh+vdvBw8xyG/GE4Kq5th50FB33S3VKRdpH2YcleggJFI7N+etHz0bDiwHbBgAbb4rAIDJkJGLN0tFWkVBUwKFDASubo62LwgCJrlL8HGCvA6yO0VXLMZPj2ocgpL091SkZQpYFKggJEOd2AnrPpLcJPnipdhfzgG5YSx4RIE50P55OCaj0gnpYBJgQJG0qquDra8V3/fzfr/Ba+FnJ7BJJ0FfSErH7Lykv7Ma2JbM+/FstLdQzlOdYZhyiLSkowMKDsteJzzdTi4u3756HVvwaYFUH0Aqg9C7eFWHD+zUQi1FFLHEFxZeZBdEPyZmatZDqRZChiRziKvF4y+OHg0VlsDNQeDsImHTvWBYF2cxtuqm9iv8Xv7tjS9H604o5FSWB1tW8FRqjH9qOqK9Lcm0hXEMiFWCDmF0X2HezAZaINAOsbgSt52aG9wv1Dj/VpVjWU1Ez7NBFd248BKoVJTNdbuFDAiEjCDrNzgEaXamrYF11GrsfBxzKyFAGvu9GFrro11nx+73aenItI5xDIhVhTtkgjuUFPVtuBqUI1Vwt4Pj3y/rvrY2xavxrJbe2qxufeSnmfmdIpqTAEjIscfS6pGiHB+uNrqJgKpNaEWPq/a3My1sWNlRwmkPJh0TTAsPkIKGBGR1oplBY/OXI0dbnQtrWpPUI1VVR79u9tIASMi0pl1VDUWAU31KiIikVDAtEJtXS3deQYEEZFU6BRZKzyx/AkeePcBJpZOTDyG9hyKdYJRGyIinYUCphXKC8sZVzKOuR/O5dnVzwLQO6c340vGJwLn5D4nk5mh/7wi0n3pJ2ArnNn/TM7sfybuzrq965i/ZT4VWyqYv2U+r6x/BYD8zHzGlYxjYulEJpRMYEy/MeTEctLcchGRjqPZlNt5NuUt+7cwf+t85m2Zx7wt81ixewUAWRlZjOk7JlHhjCsZR0FWQbt+t4hIR9B0/SnoiOn6d1ftZv7W+czfEoTOkp1LqPVaMiyDkX1GMqFkApNKJzG+dDx9crvWEEQR6Z4UMClIx3owB6oPsHDbwkTgvLvtXQ7XBZP/De05NDilVhqEzgkFJ3Ro20REUqGASUFnWHDscO1hFu1YlDiltmDrAvZX7wegf0H/xCm1CaUTGFw0WCPVRCTtFDAp6AwB01htXS0f7PqAeVvmJa7l7KzaCUCf3D71gVMygRG9RxDLiKW5xSLS3ShgUtAZA6Yxd2d15erEKbX5W+azaf8mAHpk9WB8yfjEKbVTik8hS8vkikjEFDAp6AoB05TN+zYzb+u8xGm11XtWA5ATy2Fsv7FMKJnAxNKJnNbvNPKz8tPcWhE53ihgUtBVA6axHQd3sGDrgkTgLNu1jDqvI2YxRhePTpxSm1A6gZ45PdPdXBHp4hQwKTheAqaxfYf3NRip9t7296gOF0Ya1mtYgyluSvJL0txaEelqFDApOF4DprFDtYd4b9t7iYEDC7Yu4GDNQQAGFg5MnFKbWDqRgYUDNVJNRFqkgElBdwmYxmrqali2cxkVWyoSQ6N3H9oNQL+8fkworQ+cYb2GkWGadFtE6ilgUtBdA6axOq9j1e5VzN86PxE6Ww9sBaAouyhx/WZi6URGFY8iK0Mj1US6MwVMChQwTXN3Nu7b2OBenLWVawHIy8xjbL+xQYVTMpEx/caQl5mX5haLSEdSwKRAAZO67Qe3J+7DmbdlHh/s+gDHyczI5JTiUxpM4lmUHeH65CKSdgqYFChgWq/ycCULty5MLFOwaPsiarwGwzi5z8mJgQMTSifQN69vupsrIu1IAZMCBUz7OVhzMDFSbd7Webyz9R2qaqsAGFw0uMHAgf4F/TVSTaQLU8CkQAETneraahbvXFw/xc3W+ew9vBeA0vxSLTct0oV1ioAxsxnA3UAM+LW739Ho/euAmUAtsA+41t0Xm1k28CtgElAH3OTur5pZIfC3pEOUA79396+Gx7sEuA1w4B13v7yl9ilgOk6d17F81/IGi7FtP7gdgF45vRIj1SaVTtJy0yKdXNoDxsxiwAfAdGAD8DZwmbsvTtqnyN0rw+cXATe4+wwzmwlMcvdrzKwEeA443d3rGn3HPOBmd3/NzIYDjwPnuvsuMytx960ttVEBkz7uzvq96xNhM2/LPDbs2wBouWmRzi7VgIny18TJwAp3XxU2aBZwMZAImHi4hAoIKg+A0cDscJ+tZraboJqZG985DJQS6iuaLwP3u/uu+Oci6JO0EzNjUNEgBhUN4tPDPw0cudz0vQvuBbTctEhXFWXADADWJ73eAJzReKewWrkFyAbODTe/A1wchtJAYGL459ykj14G/NHrS7AR4fHeIDgld5u7P99uvZHIlRaUcsGQC7hgyAVAsNx0fBLP+Vvn8/D7D/Pgew9quWmRLiLKgGnqqu0R5+Pc/X7gfjO7HPgecBXwMDAKqADWAm8CNY0+eilwZdLrTGA4MJXg2szfzOxUd9/doFFm1wLXAgwaNOiYOyUdp1duL6YNmsa0QdOAYLnpd7a9k6hw/vTBn/j9kt8DwXLT8ZFqWm5apHOIMmA2EFQdceXAphb2nwX8G4C71wA3x98wszeB5UmvTwMy3X1eo+97y92rgdVmtowgcN5O/hJ3fwB4AIJrMMfeLUmX/Kx8pvSfwpT+U4Ajl5t+fvXz/OcH/wnULzcdDx0tNy3S8aIMmLeB4WY2BNhIUHE0GNVlZsPdPR4cHycMETPLJxiAsN/MpgM1yYMDCE6PPdbo+/4r3P6omfUlOGW2qp37JJ1Idiyb8SXjGV8yni+N+dIRy02/sekNnln1DKDlpkXSIbKAcfcaM7sReIHgmsjD7r7IzG4HKtz9aeBGMzsfqAZ2EZweg+Di/QtmVkcQTlc2OvwlwIWNtr0AfNTMFhMMe/6Gu++Iom/SOcUyYowqHsWo4lFcMfoK3J01lWsaTHHz0tqXgGC56fhItYmlEzml+BSyY9lp7oHI8UU3WmqYcrcSX246Hjir9gRFrpabFkld2u+D6QoUMLKzaicLtiwI5lTbOp+lO5c2WG46eU41LTctElDApEABI43tO7yvwUi15pabnlAygdKC0jS3ViQ9FDApUMDI0cSXm47fALpw60IO1BwAoLxHeYM51bTctHQXCpgUKGDkWCUvNz1/y3zmb53f5HLTE0omMLz3cC03LcclBUwKFDDSVnVex+o9qxvMqbblwBYgWG56fMn4xDWc0cWjtdy0HBcUMClQwEh7c3c27d+UCJv5W+azpnINkLTcdElwSk3LTUtXpYBJgQJGOsL2g9sbrIuzbOeyBstNx5cp0HLT0lUoYFKggJF0iC83Ha9yFu1YRE1dsNz0iN4jmFg6kbH9xpKbmYuFU/oZhpkl/oxrsD0+/Z813J68X/w50OB4ljR1YOPjJd43mmxPU59r8F3x9jTafsQxm/h8U9/V4BiNjtfgc821J7lPjftrHHG8xscVBUxKFDDSGSSWm94aBM67297lYM3BdDdLmtFUeCeHKBDdLwONQ/AYQrlxQF9/2vWJmcuP+b9BJ1gPRkRSkJeZx+SyyUwumwxAdV01a/asoaauBg8nIHec4P+d+C+FHv+/pF8S468b7wcktjfeLz7HefLxPGni86aO16A9jY5/xDGTt4f7Jb/X1PPGx2vQxyaO19yxm+xHK/rY3H/bpr4rqr+r5p631Mcj2pN0/J7Z0d84rIAR6WSyMrIY3nt4upsh0mYapC8iIpFQwIiISCQUMCIiEgkFjIiIREIBIyIikVDAiIhIJBQwIiISCQWMiIhEoltPFWNm24C1rfx4X2B7OzanK1Cfuwf1uXtoS59PdPd+R9upWwdMW5hZRSpz8RxP1OfuQX3uHjqizzpFJiIikVDAiIhIJBQwrfdAuhuQBupz96A+dw+R91nXYEREJBKqYEREJBIKmCaY2QwzW2ZmK8zs1ibev8XMFpvZu2Y228xOTHrvKjNbHj6u6tiWt15r+2xm48xsjpktCt/7XMe3vnXa8vccvl9kZhvN7L6Oa3XrtfHf9SAze9HMloT7DO7ItrdWG/v80/Df9RIzu8e6yLrJKfT5OjN7z8wWmtnrZjY66b1vh59bZmYfa3Nj3F2PpAcQA1YCQ4Fs4B1gdKN9pgH54fPrgT+Gz/sAq8I/e4fPe6e7TxH3eQQwPHzeH9gM9Ep3n6Lsc9L7dwN/AO5Ld3+i7i/wKjA9fN4jvl9nfrTx3/WZwBvhMWLAHGBquvvUTn0uSnp+EfB8+Hx0uH8OMCQ8Tqwt7VEFc6TJwAp3X+Xuh4FZwMXJO7j7X9z9QPjyLaA8fP4x4CV33+nuu4CXgBkd1O62aHWf3f0Dd18ePt8EbAWOegNWJ9CWv2fMbCJQCrzYQe1tq1b3N/wNN9PdXwr325e0X2fWlr9jB3IJfkjnAFnAlg5pdduk0ufKpJcFJBa45mJglrsfcvfVwIrweK2mgDnSAGB90usN4bbmfBF4rpWf7Sza0ucEM5tM8D/Ile3aumi0us9mlgHcCXwjsta1v7b8HY8AdpvZn81sgZn9zMxiEbWzPbW6z+4+B/gLQUW+GXjB3ZdE1M72lFKfzWymma0Efgp85Vg+eywy2/Lh41RT51mbHGpnZlcAk4C/P9bPdjJt6XN8exnwO+Aqd69r9xa2v7b0+QbgWXdf30VOy0Pb+psJ/B0wHlgH/BG4Gnio3VvZvlrdZzMbBoyivqJ5yczOcffXomhoO0qpz+5+P3C/mV0OfA+4KtXPHgtVMEfaAAxMel0ObGq8k5mdD3wXuMjdDx3LZzuhtvQZMysC/gf4nru/FXFb20tb+jwFuNHM1gA/B/7RzO6Itrlt1tZ/1wvC0y41wH8BEyJub3toS58/DbwVng7cR1DZfCTi9raHY/0ZNAv4VCs/e3TpvijV2R4Ev62tIrjIFb9IdkqjfcYTnAYa3mh7H2A1wQX+3uHzPunuU8R9zgZmA19Ndz86qs+N9rmarnGRvy1/x7Fw/37h60eAmenuU8R9/hzwcniMrPDf+CfT3ad26vPwpOefBCrC56fQ8CL/Ktp4kT/t/0E64wO4EPgg/If33XDb7QS/4RD+w9sCLAwfTyd99gsEF8dWANekuy9R9xm4AqhO2r4QGJfu/kT995x0jC4RMG3tLzAdeBd4D3gUyE53f6LsM0Go/gpYAiwGfpHuvrRjn+8GFoX9/UtyABFUciuBZcAFbW2L7uQXEZFI6BqMiIhEQgEjIiKRUMCIiEgkFDAiIhIJBYyIiERCASPSzsxsXzsd5zYz+3oK+z1qZv/QHt8p0p4UMCIiEgkFjEhEzKxHuMbI/HD9jYvD7YPNbKmZ/drM3jez/zCz883sjXAdoeQZbE8zs1fC7V8OP29mdl+4jsn/ACVJ3/kDM3s7PO4DXWUNEzk+KWBEolMFfNrdJxCsO3Jn0g/8YQR3VI8FRgKXA2cDXwe+k3SMscDHCeY/+4GZ9SeYJ+tkYAzwZYK1S+Luc/fT3f1UIA/4RER9EzkqzaYsEh0D/tnMzgHqCKY+Lw3fW+3u7wGY2SJgtru7mb0HDE46xlPufhA4aGZ/IVif4xzgMXevBTaZ2StJ+08zs28C+QRz4y0CnomshyItUMCIROfzBIuvTXT36nD25dzwvUNJ+9Ulva6j4f8uG8/l5M1sx8xygV8CkzxYSuC2pO8T6XA6RSYSnZ7A1jBcpgEnHu0DTbjYzHLNrBiYCrwNvAZcamaxcB2eaeG+8TDZbmY9AI0sk7RSBSMSnf8AnjGzCoKZa5e24hhzCdbaGQT8X3ffZGZPAucSzGz8AfBXAHffbWYPhtvXEISRSNpoNmUREYmETpGJiEgkFDAiIhIJBYyIiERCASMiIpFQwIiISCQUMCIiEgkFjIiIREIBIyIikfh/c8YqugyozhEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2a6d784ff28>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_2.best_score_, gsearch2_2.best_params_))\n",
    "test_means = gsearch2_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "pd.DataFrame(gsearch2_2.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "for i, value in enumerate(reg_alpha):\n",
    "#    pyplot.plot(reg_lambda, -test_scores[i], label= 'valid_reg_alpha:'   + str(value))\n",
    "    pyplot.plot(reg_lambda, -train_scores[i], label= 'train_reg_alpha:'   + str(value))\n",
    "\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'lambda' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1单独对叶子节点数目正则系数进行调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0.8, 1, 2]}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha=[0.8,1,2]\n",
    "param_2=dict(reg_alpha=reg_alpha)\n",
    "param_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=0.8,\n",
       "        learning_rate=0.1, max_bin=256, max_depth=8, min_child_samples=20,\n",
       "        min_child_weight=1, min_split_gain=0.0, n_estimators=363,\n",
       "        n_jobs=-1, nthread=6, num_leaves=50, objective='binary',\n",
       "        random_state=None, reg_alpha=0, reg_lambda=0.25,\n",
       "        scale_pos_weight=1, seed=0, silent=True, subsample=0.8,\n",
       "        subsample_for_bin=200000, subsample_freq=1),\n",
       "       fit_params=None, iid=True, n_jobs=6,\n",
       "       param_grid={'reg_alpha': [0.8, 1, 2]}, pre_dispatch='2*n_jobs',\n",
       "       refit=True, return_train_score='warn', scoring='neg_log_loss',\n",
       "       verbose=0)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=3)\n",
    "\n",
    "params = {'boosting_type': 'gbdt', \n",
    "          'objective': 'binary', \n",
    "          'nthread': 6, \n",
    "          'silent': True,\n",
    "          'learning_rate': 0.1, \n",
    "          'num_leaves': 50, \n",
    "          'max_depth': 8,\n",
    "          'max_bin': 256, \n",
    "          'subsample_for_bin': 200000,#默认200000，用来构建直方图数据数量\n",
    "          'subsample': 0.8, #默认1\n",
    "          'subsample_freq': 1, \n",
    "          'colsample_bytree': 0.8, \n",
    "          'reg_alpha': 0,          \n",
    "          'reg_lambda': 0.25,  \n",
    "          'min_split_gain': 0.0, \n",
    "          'min_child_weight': 1,         \n",
    "          'min_child_samples': 20, \n",
    "          'scale_pos_weight': 1,\n",
    "         }\n",
    "lgbm_2=lgbm.sklearn.LGBMClassifier(n_estimators=363,seed=0,**params)\n",
    "gsearch2_2=GridSearchCV(lgbm_2,param_grid=param_2,scoring='neg_log_loss',n_jobs=6, cv=kfold)\n",
    "gsearch2_2.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mean_fit_time': array([192.56691368, 210.63699794, 128.29238534]), 'std_fit_time': array([ 0.89683706,  1.19999609, 17.03277942]), 'mean_score_time': array([22.24953945, 40.66610765, 10.79338749]), 'std_score_time': array([0.18408786, 1.11089391, 1.50064434]), 'param_reg_alpha': masked_array(data=[0.8, 1, 2],\n",
      "             mask=[False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'reg_alpha': 0.8}, {'reg_alpha': 1}, {'reg_alpha': 2}], 'split0_test_score': array([-0.39969969, -0.39979197, -0.39975447]), 'split1_test_score': array([-0.39985465, -0.39986001, -0.39987836]), 'split2_test_score': array([-0.39966234, -0.3996646 , -0.399669  ]), 'mean_test_score': array([-0.39973889, -0.39977219, -0.39976728]), 'std_test_score': array([8.32585566e-05, 8.09930858e-05, 8.59488165e-05]), 'rank_test_score': array([1, 3, 2]), 'split0_train_score': array([-0.3975134 , -0.39758671, -0.39761494]), 'split1_train_score': array([-0.3974886 , -0.39750719, -0.39758626]), 'split2_train_score': array([-0.39753379, -0.39751002, -0.39761325]), 'mean_train_score': array([-0.39751193, -0.39753464, -0.39760481]), 'std_train_score': array([1.84780309e-05, 3.68397644e-05, 1.31402038e-05])}\n",
      "-0.3997388919872223\n",
      "logloss of train : 0.398003298291121\n",
      "logloss of test : 0.40286772938280563\n"
     ]
    }
   ],
   "source": [
    "print(gsearch2_2.cv_results_)\n",
    "print(gsearch2_2.best_score_)\n",
    "y_pred_train = gsearch2_2.predict_proba(X_train)\n",
    "logloss = log_loss(y_train, y_pred_train)\n",
    "print (\"logloss of train :\",logloss )\n",
    "\n",
    "y_pred_test = gsearch2_2.predict_proba(X_test)\n",
    "logloss = log_loss(y_test, y_pred_test)\n",
    "print (\"logloss of test :\",logloss )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.8, 1, 2]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.39973889, -0.39977219, -0.39976728])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x2a6bd40d978>]"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEDCAYAAAAImhLVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3X2QVNd95vHvMzO8CgEzPYAxL0K0sSPHliU0EjPjdSI7EkLOJjgVZ2PHDiTrFGVpk01SG5eVeDfsSuUqebObFzYbeSUlZUitk00lTkR2pUUIW3ZqAZkhlkCyHAOjF8ZgNGgA8SLxNr/9ow9OC/dMj+jpuf3yfKq6+t7T5557jtDMM/ee2/cqIjAzMxtvLVl3wMzMGpMDxszMqsIBY2ZmVeGAMTOzqnDAmJlZVThgzMysKhwwZUi6T9IeSU9LelzS20eo9wVJz6bXzxeVf0jSP6byjZLaUvlnUptPp88uSuqosK8/lvZ1QdJHK2nLzKxS8vdgRidpZkS8lpb/LfDuiPj0ZXV+EvgN4E5gCvB14EPAKeAl4Cci4ruS7gVeiog/vWz7nwJ+MyI+VGFflwAzgd8CNkfEX1fSnplZJXwEU8alcEmuAkol8ruBr0fEhYg4DTwDrAJywNmI+G6qtxX42RLbfxz4i0srkj4p6Zvp6OZ/SGodY19fjIg9wPBY6puZVZMDZgwkfV7SQeATwO+WqPIMcKek6ZI6gQ8Ci4CjwCRJXaneR1N5cdvTKYTR36T164CfB94fETcAF9N+zczqSlvWHagFkp4A3lbio89FxCMR8Tngc5J+G/hVYH1xpYh4XNLNwHZgENgBXIiIkPQx4A8kTQEeBy5cto+fAv5fRAyl9Z8AbgJ2SQKYBryS+rkJWF6in38SEX/yVsdtZlZNDhggIm4bY9UvA/+HywImtfF54PMAkr4M7EvlO4APpPKVwDsv2/RjFJ0eAwRsjIjfLrGPNWPsp5lZ5nyKrAxJy4pWfxr4Tok6rZJyafl64HoKRytImpvepwCfBb5YtN0s4MeBR4qa2wZ8tGi7DknXjOeYzMwmgo9gyrtf0rsoTJy/BHwaIM2rfDoifgWYBPxDOqX1GvDJiLh0Kuwzkv4lhTB/ICK+WtT2zwCPpwsDAIiIb0v698DjklqA88C/SfseVTpN97dAO/BTkv5TRPxoBWM3M7tivkzZzMyqwqfIzMysKpr6FFlnZ2csWbIk626YmdWV3bt3H42IOeXqNXXALFmyhL6+vqy7YWZWVySVnRMGnyIzM7MqccCYmVlVOGDMzKwqHDBmZlYVDhgzM6uKigIm3cZkq6R96b19hHprU519ktYWld8kaa+k/ZI2KH0VfqR2VbAh1d8jaXkqv0bS7nR7++ckfbpUP8zMbOJUegRzD7AtIpZRuIfWPZdXSE9pXA+sAG4B1hcF0QPAOmBZeq0q0+6dRXXXpe0BDgO96fb2K4B7RnrypJmZTYxKA2Y1sDEtbwQ+UqLOHcDWiBiKiGMUHrq1StJ8YGZE7IjC/Wo2FW0/UrurgU1RsBOYLWl+RJyLiLOpzpRxGNeojp85xx89sY9nv3eimrsxM6trlf4inhcRhwHS+9wSdRYAB4vWB1LZgrR8eflo7Y7UFpIWSdqTPv9CRBwq1WFJ6yT1SeobHBwc80CLtbSIP9r2XR7/9pEr2t7MrBmUDRhJT0h6tsRr9Rj3oRJlMUr5lbRFRByMiOuBdwBrJc0r1UBEPBgRXRHRNWdO2TsdlDRz6iTeu2AWOw4cvaLtzcyaQdmAiYjbIuI9JV6PAEfSqS7S+yslmhjgzY8JXggcSuULS5QzSrsjtVXc30PAc6SHfFVLT76Tpw8e58y5yx9QaWZmUPkpss3ApavC1vLmB2ddsgVYKak9Te6vBLakU18nJXWnq8fWFG0/UrubgTXparJu4EREHJa0UNI0gLSP9wP/VOHYRtWbz3H+YtD34rFq7sbMrG5VGjD3A7dL2gfcntaR1CXpYYD0rPn7gF3pdW/R8+fvAh4G9gMHgMdGaxd4FOhP9R8C7k7l1wFPSXoG+DrwXyJib4VjG1XXknYmtYrtB16t5m7MzOpWUz9wrKurKyq5m/LPfXE75y4M88iv/otx7JWZWW2TtDsiusrV8zf5K9CT72Tv907w2hvns+6KmVnNccBUoDefYzjgm/1D5SubmTUZB0wFblw8myltLZ6HMTMrwQFTgSltrXQtaWe7vw9jZvZDHDAV6s138p3vn2To9Lmsu2JmVlMcMBXqXpoDYGe/T5OZmRVzwFTo+oWzuGpyq0+TmZldxgFToUmtLdxybQc7PNFvZvYmDphx0JPPcWDwNEdeeyPrrpiZ1QwHzDjozXcC+CjGzKyIA2YcXDd/JrOmTfI8jJlZEQfMOGhtEd1LO9jhK8nMzH7AATNOepbmODj0OgeHzmTdFTOzmuCAGSe97/A8jJlZMQfMOFk2dwadMyb7NJmZWeKAGSeS6F6aY/uBozTzM3bMzC5xwIyj3nwnR147S//R01l3xcwscw6YcdSbL9yXzPMwZmYOmHF1TW4682dNdcCYmeGAGVeS6Mnn2NH/KsPDnocxs+bmgBlnvflOhk6f47uvnMy6K2ZmmXLAjLOeNA+zfb9Pk5lZc3PAjLMFs6dxTW462z0PY2ZNzgFTBb35HE+98CoXPQ9jZk2sooCR1CFpq6R96b19hHprU519ktYWld8kaa+k/ZI2SNJo7apgQ6q/R9Lyy/YzU9L3JP1xJeOqVE++k5NvXOC5Qyey7IaZWaYqPYK5B9gWEcuAbWn9TSR1AOuBFcAtwPqiIHoAWAcsS69VZdq9s6juurR9sfuAr1c4pop1L+0A8GkyM2tqlQbMamBjWt4IfKREnTuArRExFBHHgK3AKknzgZkRsSMK91bZVLT9SO2uBjZFwU5gdmoHSTcB84DHKxxTxeZePZVlc2c4YMysqVUaMPMi4jBAep9bos4C4GDR+kAqW5CWLy8frd2SbUlqAf4r8JlyHZa0TlKfpL7BwcFy1a9Ybz5H34tDnLswXLV9mJnVsrIBI+kJSc+WeK0e4z5UoixGKb+Stu4GHo2IgyU+f3PliAcjoisiuubMmVOu+hXryec4c+4iewaOV20fZma1rK1chYi4baTPJB2RND8iDqdTVa+UqDYA3Fq0vhB4MpUvvKz8UFoeqd0BYFGJbXqAD0i6G5gBTJZ0KiJ+aE5ooqy4NodUmIfpWtKRVTfMzDJT6SmyzcClq8LWAo+UqLMFWCmpPU3urwS2pFNfJyV1p6vH1hRtP1K7m4E16WqybuBERByOiE9ExOKIWAL8FoV5mszCBaD9qsm8e/5M35fMzJpWpQFzP3C7pH3A7WkdSV2SHgaIiCEKV3ftSq97UxnAXcDDwH7gAPDYaO0CjwL9qf5DFE6N1ayepTl2v3yMN85fzLorZmYTTs38cKyurq7o6+urWvtf/c4R/vWX+vjyr6z4wSOVzczqnaTdEdFVrp6/yV9FNy/poLVFvlzZzJqSA6aKrp46iesXzmJHvwPGzJqPA6bKepbmeObgcU6dvZB1V8zMJpQDpsp6851cGA52vThUvrKZWQNxwFTZTde0M7m1hZ2ehzGzJuOAqbJpk1u5YfFsT/SbWdNxwEyA3nyOZw+d4MSZ81l3xcxswjhgJkBvvpMIeOoFH8WYWfNwwEyA9y2axdRJLT5NZmZNxQEzAaa0tXLzkg7fl8zMmooDZoL05HP805GTHD11NuuumJlNCAfMBOnNF+5FttPf6jezJuGAmSDveftMZkxp8zyMmTUNB8wEaWttYcW1nocxs+bhgJlAPfkcLxw9zeETr2fdFTOzqnPATKCefA7ARzFm1hQcMBPourfNZPb0SZ6HMbOm4ICZQC0tomdpjh0HXqWZnyRqZs3BATPBevI5vnf8dQ4OeR7GzBqbA2aC9aZ5mO0HjmbcEzOz6nLATLD8nBnMuXqKH6NsZg3PATPBpMI8zHbPw5hZg3PAZKA3n2Pw5FkODJ7KuitmZlXjgMnApfuS+XJlM2tkFQWMpA5JWyXtS+/tI9Rbm+rsk7S2qPwmSXsl7Ze0QZJGa1cFG1L9PZKWF7V1UdLT6bW5knFV26KOaSyYPc1fuDSzhlbpEcw9wLaIWAZsS+tvIqkDWA+sAG4B1hcF0QPAOmBZeq0q0+6dRXXXpe0veT0ibkivn65wXFUliZ58jh39rzI87HkYM2tMlQbMamBjWt4IfKREnTuArRExFBHHgK3AKknzgZkRsSMKs92birYfqd3VwKYo2AnMTu3Und58juNnzvP891/LuitmZlVRacDMi4jDAOl9bok6C4CDResDqWxBWr68fLR2R2oLYKqkPkk7JZUKOgAkrUv1+gYHB8cyxqrwfcnMrNGVDRhJT0h6tsRr9Rj3oRJlMUr5lbQFsDgiuoBfAP5QUr5UAxHxYER0RUTXnDlzyuyueubPmsa1nVc5YMysYbWVqxARt430maQjkuZHxOF0quqVEtUGgFuL1hcCT6byhZeVH0rLI7U7ACwqtU1EXHrvl/QkcCNwoNz4stSTz7H56UNcuDhMW6sv6DOzxlLpb7XNwKWrwtYCj5SoswVYKak9Te6vBLakU18nJXWnq8fWFG0/UrubgTXparJu4EQKoXZJUwAkdQLvB75d4diqrjef49TZCzx7yPMwZtZ4Kg2Y+4HbJe0Dbk/rSOqS9DBARAwB9wG70uveVAZwF/AwsJ/C0cZjo7ULPAr0p/oPAXen8uuAPknPAF8D7o+Img+Y7qW+L5mZNS418+1Kurq6oq+vL9M+3PEH32DuzCn8+adWZNoPM7OxkrQ7zXmPyif+M9aTz7HrxSHOXRjOuitmZuPKAZOx3nyON84P8/TB41l3xcxsXDlgMrbi2hyS52HMrPE4YDI2a/ok3vP2Wb7xpZk1HAdMDejN53j65eO8fu5i1l0xMxs3Dpga0J3Pce7iMLtfOpZ1V8zMxo0DpgbcvKSDthZ5HsbMGooDpgbMmNLG+xbNZke/52HMrHE4YGpEz9IcewZOcPKN81l3xcxsXDhgakRvPsfF4WDXi0PlK5uZ1QEHTI1Yfk07k9tafPt+M2sYDpgaMXVSK8sXz/b3YcysYThgakhvvpNvH36NY6fPZd0VM7OKOWBqSG8+RwQ89YKPYsys/jlgasj1C2czfXKr52HMrCE4YGrI5LYWupZ0eB7GzBqCA6bG9OZz7HvlFK+cfCPrrpiZVcQBU2N684XHKO/s9/dhzKy+OWBqzI++fRZXT21jh+9LZmZ1zgFTY1pbxIprc56HMbO654CpQb35HC+9eobvHX89666YmV0xB0wN6knzML5c2czqmQOmBr1r3tV0XDXZz4cxs7rmgKlBLS2iZ2mOnQdeJSKy7o6Z2RWpKGAkdUjaKmlfem8fod7aVGefpLVF5TdJ2itpv6QNkjRauyrYkOrvkbS8qK3Fkh6X9Lykb0taUsnYstaTz3HoxBu89OqZrLtiZnZFKj2CuQfYFhHLgG1p/U0kdQDrgRXALcD6oiB6AFgHLEuvVWXavbOo7rq0/SWbgN+LiOvSfl6pcGyZujQP46vJzKxeVRowq4GNaXkj8JESde4AtkbEUEQcA7YCqyTNB2ZGxI4onAfaVLT9SO2uBjZFwU5gtqT5kt4NtEXEVoCIOBURdf2n/9LOq5g3c4rnYcysblUaMPMi4jBAep9bos4C4GDR+kAqW5CWLy8frd2R2noncFzSVyR9S9LvSWot1WFJ6yT1SeobHBx8C0OdWJLozXeys9/zMGZWn8oGjKQnJD1b4rV6jPtQibIYpfxK2moDPgD8FnAzsBT4pVINRMSDEdEVEV1z5swps7ts9SzNcfTUOfa9cirrrpiZvWVt5SpExG0jfSbpiKT5EXE4nfIqNe8xANxatL4QeDKVL7ys/FBaHqndAWBRiW0mAd+KiP7Ur78DuoE/LTe+WvaDeZj9R3nnvKsz7o2Z2VtT6SmyzcClq8LWAo+UqLMFWCmpPU3urwS2pFNfJyV1p6vH1hRtP1K7m4E16WqybuBEamcX0C7p0iHJh4BvVzi2zC3qmM6ijmns6PdEv5nVn0oD5n7gdkn7gNvTOpK6JD0MEBFDwH0UQmAXcG8qA7gLeBjYDxwAHhutXeBRoD/Vfwi4O+3jIoXTY9sk7aVwKu2hCsdWE3qW5tjZP8TFYc/DmFl9UTNPIHd1dUVfX1/W3RjV333re/zG/3qa//1r/4L3LJiVdXfMzJC0OyK6ytXzN/lrnO9LZmb1ygFT4+bNnMrSOVf5+zBmVnccMHWgN5/jmy8Mcf7icNZdMTMbMwdMHejNd3L63EX2DJzIuitmZmPmgKkD3UsL8zA7fbmymdURB0wd6LhqMj/ytqs9D2NmdcUBUyd68530vXiMsxcuZt0VM7MxccDUid58jrMXhvnWy8ez7oqZ2Zg4YOrELUs7aJGfD2Nm9cMBUydmTp3EexfMYofnYcysTjhg6khPvpOnDx7nzLkLWXfFzKwsB0wd6cnnOH8x6HvxWNZdMTMrywFTR25e0k5bizwPY2Z1wQFTR6ZPbuPGxbP9fBgzqwsOmDrTk+9k78BxXnvjfNZdMTMblQOmzvQszTEc8M3+ofKVzcwy5ICpMzcuns2UthbPw5hZzXPA1Jmpk1rpWtLueRgzq3kOmDrUszTH84dfY+j0uay7YmY2IgdMHerJdwK+fb+Z1TYHTB26fuEsrprcyg7Pw5hZDXPA1KFJrS3cfG2Hnw9jZjXNAVOnevM5Dgye5shrb2TdFTOzkhwwdarX8zBmVuMqChhJHZK2StqX3ttHqLc21dknaW1R+U2S9kraL2mDJI3Wrgo2pPp7JC1P5R+U9HTR6w1JH6lkbLXuuvkzmTm1je37HTBmVpsqPYK5B9gWEcuAbWn9TSR1AOuBFcAtwPqiIHoAWAcsS69VZdq9s6juurQ9EfG1iLghIm4APgScAR6vcGw1rbVFdC/Nsb3f8zBmVpsqDZjVwMa0vBEoddRwB7A1IoYi4hiwFVglaT4wMyJ2REQAm4q2H6nd1cCmKNgJzE7tFPso8FhEnKlwbDWvN5/j4NDrHBxq+KGaWR2qNGDmRcRhgPQ+t0SdBcDBovWBVLYgLV9ePlq7I7VV7GPAX4zUYUnrJPVJ6hscHBxlaLWv9x2FeRh/q9/MalHZgJH0hKRnS7xWj3EfKlEWo5RfSVuX+jofeC+wZaQGIuLBiOiKiK45c+aU2V1tWzZ3Bp0zJvv7MGZWk9rKVYiI20b6TNIRSfMj4nD65f5KiWoDwK1F6wuBJ1P5wsvKD6XlkdodABaNsA3AvwL+NiKa4l72UpqHOXCUiCBdI2FmVhMqPUW2Gbh0Vdha4JESdbYAKyW1p8n9lcCWdOrrpKTudPXYmqLtR2p3M7AmXU3WDZy4dCot+TijnB5rRL35To68dpYXjp7OuitmZm9SacDcD9wuaR9we1pHUpekhwEiYgi4D9iVXvemMoC7gIeB/cAB4LHR2gUeBfpT/YeAuy91RNISCkc3X69wTHWlJ58D8O37zazmqHABV3Pq6uqKvr6+rLtRkYig9/6vsnxxO//9E8uz7o6ZNQFJuyOiq1w9f5O/zkmiJ59jZ/+rDA837x8LZlZ7HDANoGdpjldPn+O7r5zMuitmZj/ggGkAP5iH8W1jzKyGOGAawML26VyTm+4vXJpZTXHANIjeNA9z0fMwZlYjHDANontpjpNvXOC5Qyey7oqZGeCAaRj+PoyZ1RoHTIOYe/VUls2d4fuSmVnNcMA0kJ58jl0vDnHuwnDWXTEzc8A0kt58jjPnLrJn4HjWXTEzc8A0khXX5pDwaTIzqwkOmAbSftVkrnvbTE/0m1lNcMA0mN58jt0vH+ON8xez7oqZNTkHTIPpfUeOcxeG+ceXj2XdFTNrcg6YBnPzkg5aW+R5GDPLnAOmwVw9dRLvXTDL8zBmljkHTAPqzed45uBxTp+9kHVXzKyJOWAaUG++kwvDwa4Xh8pXNjOrEgdMA7rpmnYmtXoexsyy5YBpQNMmt3Lj4nbPw5hZphwwDao3n+O5Qyc4ceZ81l0xsyblgGlQPUtzDAc89YKPYswsGw6YBnXD4tlMndTi02RmlhkHTIOa0tbKzUs62NnvgDGzbFQUMJI6JG2VtC+9t49Qb22qs0/S2qLymyTtlbRf0gZJGq1dFWxI9fdIWl7U1n+W9Jyk54vbambdS3N85/snOXrqbNZdMbMmVOkRzD3AtohYBmxL628iqQNYD6wAbgHWFwXRA8A6YFl6rSrT7p1Fddel7ZHUC7wfuB54D3Az8OMVjq3u9abHKPsoxsyyUGnArAY2puWNwEdK1LkD2BoRQxFxDNgKrJI0H5gZETsiIoBNRduP1O5qYFMU7ARmp3YCmApMBqYAk4AjFY6t7r13wSxmTGnzPIyZZaLSgJkXEYcB0vvcEnUWAAeL1gdS2YK0fHn5aO2WbCsidgBfAw6n15aIeL5UhyWtk9QnqW9wcHDMA61Hba0trLi2g50OGDPLQNmAkfSEpGdLvFaPcR+l5kJilPK33JakdwDXAQsphNCHJP1YqQYi4sGI6IqIrjlz5pTZXf3ryefoP3qawydez7orZtZkygZMRNwWEe8p8XoEOJJOUZHeXynRxACwqGh9IXAolS8sUc4o7Y7U1s8AOyPiVEScAh4DusuNrRn0pHkY3zbGzCZapafINgOXrgpbCzxSos4WYKWk9jS5v5LCKazDwElJ3emKrzVF24/U7mZgTbqarBs4kdp5GfhxSW2SJlGY4C95iqzZXPe2mcyePskBY2YTrtKAuR+4XdI+4Pa0jqQuSQ8DRMQQcB+wK73uTWUAdwEPA/uBAxSOPEZsF3gU6E/1HwLuTuV/nbbfCzwDPBMRf1/h2BpCS4vovjbH9gOvUriWwsxsYqiZf+l0dXVFX19f1t2ouk07XuR3H3mOb3zmgyzOTc+6O2ZW5yTtjoiucvX8Tf4mcOn7MDv6j2bcEzNrJg6YJpCfM4M5V0/x92HMbEI5YJqAJHqWeh7GzCaWA6ZJ9OZzDJ48y4HB01l3xcyahAOmSfTmOwHYccDzMGY2MRwwTWJRxzQWzJ7meRgzmzAOmCYhiZ58jh39rzI87HkYM6s+B0wT6c3nOH7mPN/5/smsu2JmTcAB00Qu3Zdsu+dhzGwCOGCayPxZ07i28yrfl8zMJoQDpsn05HN884UhLlwczrorZtbgHDBNpmdpjpNnL/Dsodey7oqZNTgHTJPpXup5GDObGA6YJjPn6im8a97Vnocxs6pzwDShnnyOXS8Oce6C52HMrHocME2oJ5/jjfPDPH3weNZdMbMG5oBpQt3X5pA8D2Nm1eWAaUKzpk/iPW+f5XkYM6sqB0yT6snn+NbLx3n93MWsu2JmDcoB06R68jnOXRxm90vHsu6KmTUoB0yTunlJB20tYke/52HMrDocME1qxpQ2rl84y8+HMbOqccA0sd58J3sGTnDq7IWsu2JmDcgB08R68zkuDge7XhjKuitm1oAqChhJHZK2StqX3ttHqLc21dknaW1R+U2S9kraL2mDJI3Wrgo2pPp7JC0vausLkp5Nr5+vZFzNYvk17UxubfH3YcysKio9grkH2BYRy4Btaf1NJHUA64EVwC3A+qIgegBYByxLr1Vl2r2zqO66tD2SfhJYDtyQ9vMZSTMrHFvDmzqpleXXzPY8jJlVRVuF268Gbk3LG4Engc9eVucOYGtEDAFI2gqskvQkMDMidqTyTcBHgMdGaXc1sCkiAtgpabak+cC7ga9HxAXggqRnKITVX1U4vobXm+/kD574Lrf//tez7oqZTaAfmT+T//bxG6u6j0oDZl5EHAaIiMOS5paoswA4WLQ+kMoWpOXLy0drd6S2nqFwZPT7wHTgg8C3S3VY0joKRz8sXrx4jMNsXD9700L6B09xzg8gM2sqi9qnVX0fZQNG0hPA20p89Lkx7kMlymKU8rfcVkQ8LulmYDswCOwASl4aFREPAg8CdHV1ldtfw1swexp/+LHq/hVjZs2pbMBExG0jfSbpiKT56ShjPvBKiWoD/PPpLoCFFE55DaTl4vJDaXmkdgeARaW2iYjPA59P/foysK/c2MzMrHoqneTfDFy6Kmwt8EiJOluAlZLa0+T+SmBLOgV2UlJ3unpsTdH2I7W7GViTribrBk6kEGqVlAOQdD1wPfB4hWMzM7MKVDoHcz/wV5I+BbwM/ByApC7g0xHxKxExJOk+YFfa5t5LE/7AXcCXgGkUJvcfG61d4FHgw8B+4Azwy6l8EvAP6Srn14BPpgl/MzPLiAoXZDWnrq6u6Ovry7obZmZ1RdLuiOgqV8/f5Dczs6pwwJiZWVU4YMzMrCocMGZmVhVNPckvaRB4qYImOoFGuFNko4wDPJZa1ShjaZRxQGVjuSYi5pSr1NQBUylJfWO5kqLWNco4wGOpVY0ylkYZB0zMWHyKzMzMqsIBY2ZmVeGAqcyDWXdgnDTKOMBjqVWNMpZGGQdMwFg8B2NmZlXhIxgzM6sKB4yZmVWFA6YMSask/ZOk/ZLuKfH5Yklfk/QtSXskfTiLfo6FpD+T9IqkZ0f4XJI2pLHukbR8ovs4FmMYxydS//dI2i7pfRPdx7EqN5aiejdLuijpoxPVt7dqLGORdKukpyU9J6kmn9M9hv+/Zkn6e0nPpHH8cql6tUDSovT76fnU118vUad6P/cR4dcIL6AVOAAsBSZTeDTzuy+r8yBwV1p+N/Bi1v0eZTw/BiwHnh3h8w9TeGSCgG7gqaz7fIXj6AXa0/KdtTqOsYwl1WkFvkrhcRUfzbrPFfy7zKbwKPPFaX1u1n2+wnH8DvCFtDwHGAImZ93vEfo6H1ielq8Gvlvid1jVfu59BDO6W4D9EdEfEeeAvwRWX1YngJlpeRb//FTOmhMR36DwwzCS1cCmKNgJzE5PFK0p5cYREdsj4lha3cmbn5xaU8bwbwLwa8DfUPqJsTVjDGP5BeArEfFyql+T4xnDOAK4Oj0ocUaqW5PPn4qIwxHxj2n5JPA8sOCyalX7uXfAjG4BcLBofYAf/sf5j8AnJQ1Q+Avz1yama1UxlvHWm0/xzw+yqzuSFgA/A3wx677uQa8HAAACS0lEQVSMg3cC7ZKelLRb0pqsO3SF/hi4jsIfk3uBX4+I4Wy7VJ6kJcCNwFOXfVS1n3sHzOhUouzy67o/DnwpIhZSONT8c0n1+t91LOOtG5I+SCFgPpt1Xyrwh8BnI+Ji1h0ZB23ATcBPAncA/0HSO7Pt0hW5A3gaeDtwA/DHkmaOvkm2JM2gcBT8GxHx2uUfl9hkXH7uK31kcqMbABYVrS/kh0+BfQpYBRAROyRNpXATuZo8/C9jLOOtC5KuBx4G7oyIV7PuTwW6gL9MjwPvBD4s6UJE/F223boiA8DRiDgNnJb0DeB9FOYF6skvA/dHYQJjv6QXgB8Bvpltt0qTNIlCuPzPiPhKiSpV+7mv17+0J8ouYJmkayVNBj4GbL6szsvATwBIug6YCgxOaC/Hz2ZgTbqqpBs4ERGHs+7UWyVpMfAV4Bcjot5+eb1JRFwbEUsiYgnw18DddRouAI8AH5DUJmk6sILCnEC9Kf6Znwe8C+jPtEcjSPNEfwo8HxG/P0K1qv3c+whmFBFxQdKvAlsoXMnzZxHxnKR7gb6I2Az8O+AhSb9J4bDyl9JfNjVH0l8AtwKdac5oPTAJICK+SGEO6cPAfuAMhb/Uas4YxvG7QA74k/SX/4Wo0TvgjmEsdaPcWCLieUn/F9gDDAMPR8Sol2dnYQz/JvcBX5K0l8Lppc9GRK3ewv/9wC8CeyU9ncp+B1gM1f+5961izMysKnyKzMzMqsIBY2ZmVeGAMTOzqnDAmJlZVThgzMysKhwwZmZWFQ4YMzOriv8PeCvpClDxJvgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2a6c2771780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pyplot.plot([0.8, 1, 2],[-0.39973, -0.39979, -0.39979])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.399739 using {'reg_alpha': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAERCAYAAADIXvg5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xt0XeV55/HvT5Zt+SpZR7bxFaMDJFBCCAiw1KZNCBeTtjGZoWkaEtxOOl5JppdpJhnIZCZ0oMwiK6tNymQlKZAG3EmbzORSnCmMMQToTLETi4QAIRdfuBkb36/4KuuZP84rcyzOkWRJR/sc6fdZa69z9nve/e5ny5aes9/33XsrIjAzM6sFdVkHYGZmNlBOWmZmVjOctMzMrGY4aZmZWc1w0jIzs5rhpGVmZjXDSWuESbpN0tOSnpL0kKS5Zep9VtKzafndovIrJP0old8nqT6VfzK1+VT67ISk5iHG+utpX12Srh9KW2Zmw0G+TmtkSZoeEfvT+z8Bzo+Ij/Sq85vAvweuBSYCjwNXAAeBF4F3RcQvJd0KvBgRX+21/W8DfxYRVwwx1kXAdOATwMqI+NZQ2jMzGyqfaY2wnoSVTAFKfWs4H3g8Iroi4jXgJ8ASIAccjYhfpnqrgX9dYvvfA/6hZ0XSByX9MJ2F/Y2kcQOM9YWIeBroHkh9M7NKc9LKgKTbJb0M3AB8pkSVnwDXSposqQV4J7AA2AmMl9SW6l2fyovbnkwhwX07rZ8H/C7wqxFxEXAi7dfMrObUZx3AaCTpYeCMEh99OiLuj4hPA5+W9Cngj4BbiitFxEOSLgWeAHYAa4CuiAhJ7wc+L2ki8BDQ1Wsfvw38S0TsTuvvAi4B1kkCmARsT3GuAC4uEeeXIuJLp3vcZmaV5qRVARFx5QCr/j3wT/RKWqmN24HbAST9PbA+la8B3p7KrwbO7bXp+ynqGgQE3BcRnyqxjxsHGKeZWVVw9+AIk3RO0ep7gJ+XqDNOUi69vxC4kMJZFZJmpdeJwE3AV4q2awR+A7i/qLlHgOuLtmuWdOZwHpOZ2UjxmdbIu0PSmyhMbngR+AhAGqf6SET8ITAe+L+pO28/8MGI6OkG/KSk36LwhePLEfH9orbfCzyUJm8AEBHPSfrPwEOS6oDjwL9L++5T6qL8LjAD+G1J/zUifmUIx25mNiSe8m5mZjXD3YNmZlYz3D04zFpaWmLRokVZh2FmVlOefPLJnRExs796TlrDbNGiRXR2dmYdhplZTZHU7zg7uHvQzMxqiJOWmZnVDCctMzOrGU5aZmZWM5y0zMysZjhpmZlZzXDSMjOzmuGkVSXuf+oVvv6DAV2mYGY2ZjlpVYlVP32VLz26Ed8L0sysPCetKtHemuOVvYd5afehrEMxM6taTlpVoj3fAsATG3dlHImZWfVy0qoS+ZlTmDVtImuctMzMysosaaUn6K6WtD69zihTb1mqs17SsqLySyQ9I2mDpDuVnphYrl0V3JnqPy3p4l77mS7pFUlf7G8flSCJ9nyOJzbu8riWmVkZWZ5p3Qw8EhHnUHgk/M29K0hqBm4BLgcuA24pSm5fBpYD56RlST/tXltUd3navthtwOO9ysrtoyI68jl2HjzKhu0HK7kbM7OalWXSWgrcl97fB1xXos41wOqI2B0Re4DVwBJJc4DpEbEmCqclK4q2L9fuUmBFFKwFmlI7SLoEmA081LPjfvZRER0e1zIz61OWSWt2RGwFSK+zStSZB7xctL45lc1L73uX99VuybYk1QF/CXyyxL7L7eMUkpZL6pTUuWPHjlJVBmRB82Tmz5jkcS0zszIq+hBISQ8DZ5T46NMDbaJEWfRRPpi2PgY8EBEv9xqyGvA+IuIu4C6Atra2IQ1ItbfmeOi5bXR3B3V1FRtCMzOrSRVNWhFxZbnPJG2TNCcitqauuO0lqm0G3lG0Ph94LJXP71W+Jb0v1+5mYEGJbdqBt0v6GDAVmCDpIPDXfeyjYjrOzvG/ntzMc1v3c8G8xkrvzsyspmTZPbgS6JkNuAy4v0SdVcDVkmakCRhXA6tSt98BSYvTjL4bi7Yv1+5K4MY0i3AxsC8itkbEDRGxMCIWAZ+gMO51cz/7qJj21sK41tpN7iI0M+sty6R1B3CVpPXAVWkdSW2S7gGIiN0UZvWtS8utqQzgo8A9wAZgI/BgX+0CDwCbUv27KXQL9qfcPirmjMYGWlumeDKGmVkJ8jVBw6utrS06OzuH1Manv/sM9z+1hac+cxX143z9t5mNfpKejIi2/ur5L2IV6si3cPBoF8+8si/rUMzMqoqTVhVa3NoM+HotM7PenLSqUG7qRN58xjRfr2Vm1ouTVpVqz+dY98JujnadyDoUM7Oq4aRVpTryLRzt6uapl/ZmHYqZWdVw0qpSl53VTJ08rmVmVsxJq0o1ThrPBfMaPa5lZlbESauKtedz/PjlPRw+5nEtMzNw0qpq7a05jp8IOl/c3X9lM7MxwEmril26qJn6Onlcy8wscdKqYlMm1nPRgiaPa5mZJU5aVa49n+PpzXvZf+R41qGYmWXOSavKtedzdAese97jWmZmTlpV7uKFM5hQX+dxLTMznLSqXsP4cVyycIbHtczMcNKqCR35HM9t3c+e145lHYqZWaactGpAx9k5ANZu8tmWmY1tTlo14ML5TUyeMI41TlpmNsY5adWA8ePquHRRsydjmNmY56RVIzryOTZsP8j2/UeyDsXMLDOZJC1JzZJWS1qfXmeUqbcs1VkvaVlR+SWSnpG0QdKdktRXuyq4M9V/WtLFvfYzXdIrkr5YVPaYpF9Ieiotsyrz0xiYjnwLgLsIzWxMy+pM62bgkYg4B3gkrZ9CUjNwC3A5cBlwS1Fy+zKwHDgnLUv6affaorrL0/bFbgMeLxHnDRFxUVq2D+ZAh8v5c6czvaHeU9/NbEzLKmktBe5L7+8DritR5xpgdUTsjog9wGpgiaQ5wPSIWBMRAawo2r5cu0uBFVGwFmhK7SDpEmA28NCwHuEwG1cnLm/NeVzLzMa0rJLW7IjYCpBeS3W9zQNeLlrfnMrmpfe9y/tqt2RbkuqAvwQ+WSbOr6Wuwf/S0wVZiqTlkjolde7YsaNctSHryOd4afchNu85VLF9mJlVs4olLUkPS3q2xLJ0oE2UKIs+ygfT1seAByLi5RKf3xARbwHenpYPlWs8Iu6KiLaIaJs5c2Y/oQxee75wvZa7CM1srKqvVMMRcWW5zyRtkzQnIrambrpS40WbgXcUrc8HHkvl83uVb0nvy7W7GVhQYpt24O2SPgZMBSZIOhgRN0fEK+k4Dkj6ewrjaisGcOgVc+6saeSmTGDNxl38TtuC/jcwMxtlsuoeXAn0zAZcBtxfos4q4GpJM9IEjKuBVanb74CkxanL7sai7cu1uxK4Mc0iXAzsi4itEXFDRCyMiEXAJyiMe90sqV5SC4Ck8cBvAc8O3+EPTl2dWJwvjGsVhvPMzMaWrJLWHcBVktYDV6V1JLVJugcgInZTmNW3Li23pjKAjwL3ABuAjcCDfbULPABsSvXvptAt2JeJwCpJTwNPAa+k7TLXkc/x6v4jvLDL41pmNvbI39iHV1tbW3R2dlas/U07DnLFXz7O7e+9gBsuP7Ni+zEzG0mSnoyItv7q+Y4YNeaslimcMb3BU9/NbExy0qoxkujI51jrcS0zG4OctGrQ4nyOXa8d45fbDmYdipnZiHLSqkEd6XqtJzbuzDgSM7OR5aRVg+bPmMzC5ske1zKzMcdJq0a1t+b4waZdnOj2uJaZjR1OWjWq4+wc+4908dyW/VmHYmY2Ypy0alR7q8e1zGzscdKqUbOmN3D2rKl+KKSZjSlOWjWsvTXHD5/fzfET3VmHYmY2Ipy0alhHPsehYyd4evPerEMxMxsRTlo1bHGrn69lZmOLk1YNmzFlAufNme7rtcxszHDSqnEd+RydL+7hyPETWYdiZlZxTlo1riOf41hXNz96aU/WoZiZVZyTVo279Kxm6gRr3UVoZmOAk1aNm94wnrfMb/K4lpmNCU5ao0BHPsdTL+/ltaNdWYdiZlZRTlqjQHtrjq7uoPNFj2uZ2ejmpDUKtC2awfhx8n0IzWzUc9IaBSZPqOdtC2b4ImMzG/UySVqSmiWtlrQ+vc4oU29ZqrNe0rKi8kskPSNpg6Q7JamvdlVwZ6r/tKSLi9o6IemptKwsKj9L0g9SW9+UNKFyP5Gha8/nePaVfew7fDzrUMzMKiarM62bgUci4hzgkbR+CknNwC3A5cBlwC1Fye3LwHLgnLQs6afda4vqLk/b9zgcERel5T1F5Z8FPp/a2gN8eGiHXFnt+RzdAT98fnfWoZiZVUxWSWspcF96fx9wXYk61wCrI2J3ROwBVgNLJM0BpkfEmogIYEXR9uXaXQqsiIK1QFNqp6R05nYF8K1+Yqwab1vYxMT6Oo9rmdmollXSmh0RWwHS66wSdeYBLxetb05l89L73uV9tVuuLYAGSZ2S1krqSUw5YG9EdJWo/waSlqc2Onfs2FGuWkVNrB/HpYuaPa5lZqNaxZKWpIclPVtiWTrQJkqURR/lg2kLYGFEtAEfAL4gKX+6+4iIuyKiLSLaZs6c2U8oldOez/HzVw+w6+DRzGIwM6ukiiWtiLgyIi4osdwPbOvpnkuv20s0sRlYULQ+H9iSyueXKKePdsu1RUT0vG4CHgPeBuyk0IVYX2IfVas9X3hUydpNHtcys9Epq+7BlUDPbMBlwP0l6qwCrpY0I03AuBpYlbr9DkhanMaebizavly7K4Eb0yzCxcC+iNia2p4IIKkF+FXguTRW9ihwfT8xVpUL5zUydWK9x7XMbNTKKmndAVwlaT1wVVpHUpukewAiYjdwG7AuLbemMoCPAvcAG4CNwIN9tQs8AGxK9e8GPpbKzwM6Jf2EQpK6IyKeS5/dBHxc0gYKY1xfHdafQAXUj6vj0kUzWLPJ41pmNjqpcFJhw6WtrS06Ozsz2//d/7yJ2x/4GWs/9S7OaGzILA4zs9Mh6ck0v6BPviPGKNMzrrVmk7sIzWz0cdIaZc6fM53GSeM99d3MRiUnrVGmrk4sbm3287XMbFRy0hqFOvItbN5zmJd3H8o6FDOzYeWkNQp1pHEtT303s9HGSWsUOnvWVFqmTvS4lpmNOk5ao5Ak2vM5nti4C1/SYGajSb9JS1K+6K4R75D0J5KaKh+aDUVHPsf2A0fZuOO1rEMxMxs2AznT+jZwQtLZFO4KcRbw9xWNyoasvbXnei13EZrZ6DGQpNWdHtHxXuALEfFnQNlnUVl1ODM3mbmNDazxZAwzG0UGkrSOS/o9CjeN/d+pbHzlQrLhUBjXamHNxl10d3tcy8xGh4EkrT8A2oHbI+J5SWcB/6OyYdlwaM/n2HPoOL/YdiDrUMzMhkV9fxXSXc//BCA9ImRaRNzR91ZWDdpPXq+1i/PmTM84GjOzoRvI7MHHJE2X1Az8BPiapL+qfGg2VPOaJrEoN9njWmY2agyke7AxIvYD/wr4WkRcAlxZ2bBsuLTnW/jBpt10nejOOhQzsyEbSNKqT4+ufx+vT8SwGtGez3HgaBc/3bI/61DMzIZsIEnrVmAVsDEi1klqBdZXNiwbLj3Xa/mu72Y2GvSbtCLif0XEhRHx0bS+KSL+deVDs+Ewc9pEzp091TfPNbNRYSATMeZL+q6k7ZK2Sfq2pPkjEZwNj/bWHJ0v7OFYl8e1zKy2DaR78GvASmAuMA/4XiqzGtGeb+Hw8RP8ZPPerEMxMxuSgSStmRHxtYjoSsu9wMwKx2XDaHFrMxI8scHjWmZW2waStHZK+qCkcWn5IDCkv36SmiWtlrQ+vc4oU29ZqrNe0rKi8kskPSNpg6Q7JamvdlVwZ6r/tKSLi9o6IemptKwsKr9X0vNFn100lGPOUtPkCZw/ZzprNnlcy8xq20CS1r+hMN39VWArcD2FWzsNxc3AIxFxDvBIWj9Fupj5FuBy4DLglqLk9mVgOXBOWpb00+61RXWXp+17HI6Ii9Lynl5hfLLos6eGdMQZ68jn+NGLezly/ETWoZiZDdpAZg++FBHviYiZETErIq6jcKHxUCwF7kvv7wOuK1HnGmB1ROyOiD3AamBJumZsekSsicITDlcUbV+u3aXAiihYCzSldsaMjnwLx0508+SLe7IOxcxs0Ab75OKPD3G/syNiK0B6nVWizjzg5aL1zalsXnrfu7yvdsu1BdAgqVPSWkm9k+ftqTvx8z0PwixF0vLURueOHTvKVcvUpWc1M65OrPH1WmZWwwabtNRvBelhSc+WWJYOYR/RR/lg2gJYGBFtwAeAL0jKp/JPAW8GLgWagZvKNR4Rd0VEW0S0zZxZnXNUpk6s58L5jb5ey8xq2mCTVr8PaIqIKyPighLL/cC2nu659Lq9RBObgQVF6/OBLal8foly+mi3XFtERM/rJuAx4G1pfWvqTjxKYYr/Zf0dc7XryOf4yeZ9HDzalXUoZmaDUjZpSTogaX+J5QCFa7aGYiWFh0qSXu8vUWcVcLWkGWkCxtXAqtTtd0DS4jRr8Mai7cu1uxK4Mc0iXAzsi4itqe2J6XhbgF8FnkvrPclPFMbGnh3iMWeuI9/Cie5g3fO7sw7FzGxQyj5PKyKmVXC/dwD/U9KHgZeA3wGQ1AZ8JCL+MCJ2S7oNWJe2uTUiev7afhS4F5gEPJiWsu0CDwDvBjYAh3h99uN5wN9I6qaQwO9Izw8D+LqkmRS6Fp8CPjKMx5+JS86cwYRxdazZtIt3vrnUMKKZWXVTYQKeDZe2trbo7OzMOoyyfvdv1vDasS7+9x+/PetQzMxOkvRkml/Qp8GOaVmN6si38NMt+9l76FjWoZiZnTYnrTGmPZ8jAn7gcS0zq0FOWmPMRQuaaBhf5+u1zKwmlZ2I0SPNFuw98LUP6AT+Q5oqbjViQn0dly5q9vVaZlaTBnKm9VfAJyncQWI+8AngbuAbwN9WLjSrlPZ8jl9uO8iOA0ezDsXM7LQMJGktiYi/iYgDEbE/Iu4C3h0R3wRK3p3dqltHvgWAtZvcRWhmtWUgSatb0vsk1aXlfUWfeb58Dbpg7nSmTaznCY9rmVmNGUjSugH4EIVbIm1P7z8oaRLwRxWMzSqkflwdl7c2s8bjWmZWY/qdiJEmWvx2mY//3/CGYyNlcWuOh3+2nS17DzO3aVLW4ZiZDUi/Z1qS5kv6rqTtkrZJ+rak+f1tZ9WtZ1zLU9/NrJYMpHvwaxRuODuXwgzC76Uyq2FvPmMaMyaP97iWmdWUgSStmRHxtYjoSsu9QHU+NMoGrK5OLG7NsXbTLnz/STOrFQNJWjslfVDSuLR8EPDX81GgI5/jlb2HeWn3oaxDMTMbkIEkrX8DvA94FdgKXM/rj/awGtaexrXcRWhmtaLfpBURL0XEeyJiZkTMiojrgH81ArFZheVnTmHmtImejGFmNWOwN8z9+LBGYZmQREc+xxMbPa5lZrVhsElLwxqFZaYjn2PnwaNs2H4w61DMzPo12KTlr+WjRIfHtcyshpRNWpIOSNpfYjlA4ZotGwUWNE9mXtMkj2uZWU0oexuniJg2koFYdjryOR56bhvd3UFdnXt+zax6ZfLkYknNklZLWp9eSz7iRNKyVGe9pGVF5ZdIekbSBkl3SlJf7argzlT/aUkXF7W1UNJDkn4m6TlJi1L5WZJ+kNr6pqQJlfyZZKnj7Bz7Dh/nua37sw7FzKxPmSQt4GbgkYg4B3gkrZ9CUjNwC3A5cBlwS1Fy+zKwHDgnLUv6affaorrL0/Y9VgCfi4jz0n62p/LPAp9Pbe0BPjzEY65a7a1+vpaZ1YasktZS4L70/j7guhJ1rgFWR8TuiNgDrAaWSJoDTI+INVGYp72iaPty7S4FVkTBWqBJ0hxJ5wP1EbEaICIORsShdOZ2BfCtfmIcFc5obKC1ZYonY5hZ1csqac2OiK0A6XVWiTrzgJeL1jensnnpfe/yvtot19a5wF5J35H0Y0mfkzQOyAF7I6KrxD5GpfZ8jh8+v5uuE91Zh2JmVlbFkpakhyU9W2JZOtAmSpRFH+WDaaseeDvwCeBSoBX4/dPdh6Tlkjolde7YsaOfUKpTez7HwaNdPPPKvqxDMTMrq2JJKyKujIgLSiz3A9tSNx/pdXuJJjYDC4rW5wNbUvn8EuX00W5fbf04Ijals6p/BC4GdlLoQqzvVb/csd4VEW0R0TZzZm3eAH9xaw7w9VpmVt2y6h5cCfTMBlwG3F+izirgakkz0gSMq4FVqdvvgKTFaezpxqLty7W7ErgxzSJcDOxL7awDZkjqyTRXAM+lsbJHKdwcuK8YR42WqRN58xnTfL2WmVW1rJLWHcBVktYDV6V1JLVJugcgInYDt1FILOuAW1MZwEeBe4ANwEbgwb7aBR4ANqX6dwMfS/s4QaFr8BFJz1DoFrw7bXMT8HFJGyiMcX11mH8GVWdxa451L+zmaNeJrEMxMytJvlHq8Gpra4vOzs6swxiUh376Ksv/7km+uXwxl6fuQjOzkSDpyYho669eVmdaVoUub81RJ49rmVn1ctKykxonjeeCeY0e1zKzquWkZadob83x45f3cPiYx7XMrPo4adkp2vM5jp8IOl/c3X9lM7MR5qRlp7h0UTP1dfK4lplVJSctO8WUifW8dUGTx7XMrCo5adkbdORzPL15L/uPHM86FDOzUzhp2Ru053N0B6x73uNaZlZdnLTsDS5eOIMJ9XUe1zKzquOkZW/QMH4clyyc4XEtM6s6TlpWUkc+x3Nb97PntWNZh2JmdpKTlpXUcXbh3oNrN/lsy8yqh5OWlXTh/CYmTxjHGictM6siTlpW0vhxdVy6qNmTMcysqjhpWVkd+Rwbth9k+/4jWYdiZgY4aVkf2vOFcS13EZpZtXDSsrJ+ZW4j0xrqPfXdzKqGk5aVNa5OLG7NeVzLzKqGk5b1qb01x0u7D7F5z6GsQzEzc9KyvvVcr+UuQjOrBk5a1qdzZ00jN2WCk5aZVYVMkpakZkmrJa1PrzPK1FuW6qyXtKyo/BJJz0jaIOlOSeqrXRXcmeo/LeniorYWSnpI0s8kPSdpUSq/V9Lzkp5Ky0WV/JlUq7o6sThfGNeKiKzDMbMxLqszrZuBRyLiHOCRtH4KSc3ALcDlwGXALUXJ7cvAcuCctCzpp91ri+ouT9v3WAF8LiLOS/vZXvTZJyPiorQ8NbRDrl3trTle3X+EF3Z5XMvMspVV0loK3Jfe3wdcV6LONcDqiNgdEXuA1cASSXOA6RGxJgpf/VcUbV+u3aXAiihYCzRJmiPpfKA+IlYDRMTBiPBf5l460vVaT2zcmXEkZjbWZZW0ZkfEVoD0OqtEnXnAy0Xrm1PZvPS+d3lf7ZZr61xgr6TvSPqxpM9JGldU7/bUnfh5SRPLHYyk5ZI6JXXu2LGj7yOvQWe1TOGM6Q2e+m5mmatY0pL0sKRnSyxLB9pEibLoo3wwbdUDbwc+AVwKtAK/nz7/FPDmVN4M3FSu8Yi4KyLaIqJt5syZ/YRSeyTRns+x1uNaZpaxiiWtiLgyIi4osdwPbEvdfKTX7SWa2AwsKFqfD2xJ5fNLlNNHu3219eOI2BQRXcA/Ahen+Lem7sSjwNcojHeNWe35HLteO8Yvtx3MOhQzG8Oy6h5cCfTMBlwG3F+izirgakkz0gSMq4FVqdvvgKTFadbgjUXbl2t3JXBjmkW4GNiX2lkHzJDUc3p0BfAcnEx6pH1cBzw7DMddszyuZWbVIKukdQdwlaT1wFVpHUltku4BiIjdwG0UEss64NZUBvBR4B5gA7AReLCvdoEHgE2p/t3Ax9I+TlDoGnxE0jMUuhHvTtt8PZU9A7QAfzHMP4OaMn/GZBY0T/K4lpllSh6jGF5tbW3R2dmZdRgVcdO3nubBZ7fy489czbi6UsOEZmaDI+nJiGjrr57viGED1nF2jv1Hunhuy/6sQzGzMcpJywasvdXjWmaWLSctG7BZ0xvIz5zih0KaWWactOy0dORb+OHzuzl+ojvrUMxsDHLSstPSkc9x6NgJnt68N+tQzGwMctKy03J5q5+vZWbZcdKy09I8ZQLnzZnu67XM7BSvHe2iu7vyl1DVV3wPNup05HP83doXOXL8BA3jx/W/gZnVtOMnunl13xG27D3M1n1HeGXvYbbuO8yWvYWyLXsPs/9IF49/8h2cmZtS0VictOy0tbfm+Or/e54fvbSHjnxL1uGY2RB0dwe7XjuWEtJhXtl7hK17D7MlJaWt+w6z/cBRet+HYsbk8cxpnMT8GZO57Kxm5jROYsrEyqcUJy07bZe1NlMnWLtxl5OWWZU7cOT462dHPWdG+w6fPGvauvcIx3rNBm4YX8fcpknMbZzEb5w7kzmNk5jXNIk5TQ3MbZrEnMYGJk/IJn04adlpm94wnrfMb+KJjbv4eNbBmI1hR7tOsG3f0aLuusNs6enGSwnqwNGuU7YZVyfOmN7AnMYG3jq/iSUXNBQSUuMk5jY1MLdxEk2Tx1O4V3j1cdKyQWlvzXHP/93Ea0e7RqRLwGys6e4Odh48ejIJFZYjpySnHQeOvmG73JQJzGlqYGFuMu35HHMaC2dHc9NZ0sypE6kfV7tz8PzXxgalI5/jK49vpPPFPfzGuaPvwZdmlRQR7D/S9XoC2vvGSQ6v7jvC8ROnDiRNnjDuZPfceXOmv3521DTpZPlonxzlpGWD0rZoBuPHiSc27nTSMuvlyPEThdl2PZMZiiY29CSng7267errxBmNhe65SxbOYE7TJOY29owhFcaUpk+qr9puu5HipGWDMnlCPW9bMMMXGduYc6I72HHg6OuTGfaeOgV8677D7Dx47A3btUydwNymSbTOnMKvndPC3MZ0dtRUGFNqmTrRj/wZACctG7TF+Rxf/P569h0+TuOk8VmHYzZkEcG+w8eLzohOndjwyt7DbNt/hK5eF9FOnVjP3KYG5jRO4oJ5jcxtbCicKaWJDWeMgW67keKkZYPWkc9x5yPr+eHzu7nq/NlZh2PWryPHT5w6dlRiCvihYydO2Wb8uNe77S47q/lkciqeAj69wV/aRoqTlg3a2xY2MbG+jic27nTSssx1nehm+4Gjp1wg2/vuDbtusQ8ZAAAL0UlEQVRfe2O33cxpE5nbNIlzZ0/jHW+axZzGNAU8jSm1TJ1InbvtqoaTlg3axPpxtC3yuJZVXkSw99DxlICKz47SJIe9h9l24CgnenXbTWuoT2NHDVw4vyldj9Rw8sLZ2Y0TmVjvbrta4qRlQ9KRb+Fzq37BroNHyU2dmHU4VqMOHes69Rqkoll2PV13R46feteGCePqCt1zjZNYnM+deoFsSk7T3G036jhp2ZC05wuPKlm7aTe/eeGcjKOxatR1opttB46WvkB2byEp7T10/JRtJJiVuu3OO2M6V7xp1ikXyM5pnERuygR3241BmSQtSc3AN4FFwAvA+yJiT4l6y4D/nFb/IiLuS+WXAPcCk4AHgD+NiCjXrgoXNvw18G7gEPD7EfEjSe8EPl+0yzcD74+If5R0FvANoBn4EfChiHhjh/gY95Z5jUyZMI4nNu500hqDIoLdrx07mXxOneRQSErbDxyh9xMrGieNPzl2dPGZTSe763rOkGZPb2BCfe3etcEqJ6szrZuBRyLiDkk3p/WbiiukBHQL0AYE8KSklSm5fRlYDqylkLSWAA/20e61wDlpuTxtf3lEPApcVLS/DcBDKYTPAp+PiG9I+grw4bSdFRk/ro7LzmpmzSaPa41Grx3tOnnLoJ6xoy1Fj6jYsvcwR7tO7babWF938qzo185pSQmpMAV8Xpp551t/2WBl9T9nKfCO9P4+4DF6JS3gGmB1ROwGkLQaWCLpMWB6RKxJ5SuA6ygkrXLtLgVWREQAayU1SZoTEVuL9nc98GBEHEpnZlcAHyhq689x0iqpI9/Co7/4Ga/uO8IZjQ1Zh2MDNNBnJBWrE8xON1v9lbnTuer82UUJqXCW1Dxlwpi/a4NVTlZJa3ZPwoiIrZJmlagzD3i5aH1zKpuX3vcu76vdcm0VJ633A3+V3ueAvRHR1at+SZKWUzjzY+HCheWqjVo941prNu3kvW+bn3E0Bm98RlKpe9sN5BlJc4tn2zVNYva02r7ZqtW+iiUtSQ8DZ5T46NMDbaJEWfRRPpi2Ch9Kc4C3AKsGUv8NH0TcBdwF0NbWVvnnTVeZ8+ZMp3HSeNZs3OWkNUJG2zOSzAaqYv9DI+LKcp9J2tbTPZcSxvYS1TbzelcfwHwK3X2b0/vi8i3pfbl2NwMLymwD8D7guxHRM4VpJ9AkqT6dbfWub0XG1YnFrc084eu1hkW5ZyRtLZptd+DI6HpGktlAZfW1aiWwDLgjvd5fos4q4L9JmpHWrwY+FRG7JR2QtBj4AXAj8N/7aXcl8EeSvkFhIsa+XuNZvwd8qmclzUR8lMI41zf6iNGSjnwLq366jZd3H2JB8+Ssw6lafkaS2dBklbTuAP6npA8DLwG/AyCpDfhIRPxhSk63AevSNrf2TMoAPsrrU94fTEvZdinMMHw3hdmBh4A/6AlE0iIKZ2GP94rxJuAbkv4C+DHw1SEf9Sh2clxr464xm7T8jCSzylP0Hom1IWlra4vOzs6swxhxEcGltz/Mr53dwhfe/7asw6mIoT4jaW5Tg5+RZFaGpCcjoq2/eh51tWEhifZ8C09s3EVE1NwfYT8jyaw2OGnZsGlvzfG9n2xh447XOHvW1KzDOcnPSDIbPZy0bNh0nLxea9eIJi0/I8ls7HDSsmFzZm4ycxsbWLNxJx9afOawtFn8jKRSExv8jCSzscVJy4aNJBbnczz68+10d0e/ScHPSDKz0+WkZcOqI9/Cd370Cr/YdoAzc5NPTmLoPbHBz0gys8Fw0rJh1XO91nu/9C9vSEh+RpKZDZWTlg2reU2T+NN3ncOu1476GUlmNuyctGzY/dlV52YdgpmNUv7aa2ZmNcNJy8zMaoaTlpmZ1QwnLTMzqxlOWmZmVjOctMzMrGY4aZmZWc1w0jIzs5rhJxcPM0k7gBcHuXkLsHMYw8nSaDmW0XIc4GOpVqPlWIZ6HGdGxMz+KjlpVRFJnQN53HQtGC3HMlqOA3ws1Wq0HMtIHYe7B83MrGY4aZmZWc1w0qoud2UdwDAaLccyWo4DfCzVarQcy4gch8e0zMysZvhMy8zMaoaTlpmZ1QwnrQxIWiLpF5I2SLq5xOcLJT0q6ceSnpb07izi7I+kv5W0XdKzZT6XpDvTcT4t6eKRjnGgBnAsN6RjeFrSE5LeOtIxDkR/x1FU71JJJyRdP1Kxna6BHIukd0h6StJPJT0+kvGdjgH8/2qU9D1JP0nH8gcjHeNASFqQ/jb9LMX5pyXqVPb3PiK8jOACjAM2Aq3ABOAnwPm96twFfDS9Px94Ieu4yxzLrwMXA8+W+fzdwIOAgMXAD7KOeQjH0gHMSO+vrdZj6e84Up1xwPeBB4Drs455CP8mTcBzwMK0PivrmIdwLP8J+Gx6PxPYDUzIOu4Scc4BLk7vpwG/LPH3q6K/9z7TGnmXARsiYlNEHAO+ASztVSeA6el9I7BlBOMbsIj4Zwq/XOUsBVZEwVqgSdKckYnu9PR3LBHxRETsSatrgfkjEthpGsC/CcAfA98Gtlc+osEbwLF8APhORLyU6lft8QzgWAKYJknA1FS3ayRiOx0RsTUifpTeHwB+BszrVa2iv/dOWiNvHvBy0fpm3viP/ufAByVtpvBt+I9HJrRhN5BjrUUfpvBNsuZImge8F/hK1rEMg3OBGZIek/SkpBuzDmgIvgicR+EL6jPAn0ZEd7Yh9U3SIuBtwA96fVTR33snrZGnEmW9rzv4PeDeiJhP4VT77yTV4r/VQI61pkh6J4WkdVPWsQzSF4CbIuJE1oEMg3rgEuA3gWuA/yLp3GxDGrRrgKeAucBFwBclTe97k+xImkrhbP3fR8T+3h+X2GTYfu/rh6shG7DNwIKi9fm8sfvvw8ASgIhYI6mBws0oq7b7o4yBHGvNkHQhcA9wbUTsyjqeQWoDvlHohaIFeLekroj4x2zDGpTNwM6IeA14TdI/A2+lMM5Sa/4AuCMKg0IbJD0PvBn4YbZhvZGk8RQS1tcj4jslqlT0974Wv73XunXAOZLOkjQBeD+wsledl4B3AUg6D2gAdoxolMNjJXBjmk20GNgXEVuzDmowJC0EvgN8KCJq8Y8iABFxVkQsiohFwLeAj9VowgK4H3i7pHpJk4HLKYyx1KLi3/nZwJuATZlGVEIac/sq8LOI+Ksy1Sr6e+8zrREWEV2S/ghYRWEW199GxE8l3Qp0RsRK4D8Ad0v6Mwqn1b+fvoFVFUn/ALwDaEnjb7cA4wEi4isUxuPeDWwADlH4NlmVBnAsnwFywJfSWUpXVOGduQdwHDWjv2OJiJ9J+j/A00A3cE9E9DnVPysD+He5DbhX0jMUutduiohqfFzJrwIfAp6R9FQq+0/AQhiZ33vfxsnMzGqGuwfNzKxmOGmZmVnNcNIyM7Oa4aRlZmY1w0nLzMxqhpOWWQ2QdHCY2vlzSZ8YQL17q/kO8DZ2OWmZmVnNcNIyqyGSpkp6RNKPJD0jaWkqXyTp55LukfSspK9LulLSv0haL+myombeKun7qfzfpu0l6YuSnpP0T8Cson1+RtK61O5d6a4IZplw0jKrLUeA90bExcA7gb8sSiJnA38NXEjhvnUfAH4N+ASFuxb0uJDCTWbbgc9Imkvhzu9vAt4C/FsKzw/r8cWIuDQiLgAmAb9VoWMz65dv42RWWwT8N0m/TuHWRfOA2emz5yPiGQBJPwUeiYhItwZaVNTG/RFxGDgs6VEKz3j7deAf0t3ft0j6flH9d0r6j8BkoBn4KfC9ih2hWR+ctMxqyw0Unmx7SUQcl/QChRsqAxwtqtddtN7Nqb/rve/dFmXKSU8Y+BLQFhEvS/rzov2ZjTh3D5rVlkZge0pY7wTOHEQbSyU1SMpRuInrOuCfgfdLGpeeMvvOVLcnQe1Mz1DyjELLlM+0zGrL14HvSeqk8NDAnw+ijR8C/0Thzty3RcQWSd8FrqDw1NxfAo8DRMReSXen8hcoJDizzPgu72ZmVjPcPWhmZjXDScvMzGqGk5aZmdUMJy0zM6sZTlpmZlYznLTMzKxmOGmZmVnN+P8q8gWBkzGOnwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2a6c9dc6eb8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_2.best_score_, gsearch2_2.best_params_))\n",
    "test_means = gsearch2_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "pd.DataFrame(gsearch2_2.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means)\n",
    "train_scores = np.array(train_means)\n",
    "\n",
    "\n",
    "pyplot.plot(reg_alpha, test_scores, label= 'reg_alpha:' )\n",
    "#    pyplot.plot(reg_lambda, -train_scores[i], label= 'train_reg_alpha:'   + str(value))\n",
    "\n",
    "    \n",
    "#pyplot.legend()\n",
    "pyplot.xlabel( 'lambda' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4、调整行采样和列采样的比例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': [0.4, 0.6, 0.8], 'subsample': [0.4, 0.6, 0.8]}"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subsample=[0.4,0.6,0.8]\n",
    "colsample_bytree=[0.4,0.6,0.8]\n",
    "param_4=dict(subsample=subsample,colsample_bytree=colsample_bytree)\n",
    "param_4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=0.8,\n",
       "        learning_rate=0.1, max_bin=256, max_depth=8, min_child_samples=20,\n",
       "        min_child_weight=1, min_split_gain=0.0, n_estimators=363,\n",
       "        n_jobs=-1, nthread=6, num_leaves=50, objective='binary',\n",
       "        random_state=None, reg_alpha=1, reg_lambda=0.25,\n",
       "        scale_pos_weight=1, seed=0, silent=True, subsample=0.8,\n",
       "        subsample_for_bin=200000, subsample_freq=1),\n",
       "       fit_params=None, iid=True, n_jobs=6,\n",
       "       param_grid={'subsample': [0.4, 0.6, 0.8], 'colsample_bytree': [0.4, 0.6, 0.8]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=3)\n",
    "\n",
    "params = {'boosting_type': 'gbdt', \n",
    "          'objective': 'binary', \n",
    "          'nthread': 6, \n",
    "          'silent': True,\n",
    "          'learning_rate': 0.1, \n",
    "          'num_leaves': 50, \n",
    "          'max_depth': 8,\n",
    "          'max_bin': 256, \n",
    "          'subsample_for_bin': 200000,#默认200000，用来构建直方图数据数量\n",
    "          'subsample': 0.8, #默认1\n",
    "          'subsample_freq': 1, \n",
    "          'colsample_bytree': 0.8, \n",
    "          'reg_alpha': 0.8,          \n",
    "          'reg_lambda': 0.25,  \n",
    "          'min_split_gain': 0.0, \n",
    "          'min_child_weight': 1,         \n",
    "          'min_child_samples': 20, \n",
    "          'scale_pos_weight': 1,\n",
    "         }\n",
    "lgbm_2=lgbm.sklearn.LGBMClassifier(n_estimators=363,seed=0,**params)\n",
    "gsearch2_2=GridSearchCV(lgbm_2,param_grid=param_4,scoring='neg_log_loss',n_jobs=6, cv=kfold)\n",
    "gsearch2_2.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mean_fit_time': array([180.55394785, 258.98511044, 322.12442772, 235.97703187,\n",
      "       278.56989423, 292.47091881, 276.41089598, 307.3526427 ,\n",
      "       218.56975738]), 'std_fit_time': array([29.38156317,  1.31334741, 56.59163823, 27.07649671,  4.13636249,\n",
      "        8.15903006,  5.15506652,  2.69134724,  9.31214205]), 'mean_score_time': array([17.05652157, 27.34910909, 13.43893059, 16.893068  , 12.68542147,\n",
      "       13.15065996, 12.9688789 , 10.14399298, 11.82179101]), 'std_score_time': array([3.27383635, 2.09634619, 2.88690092, 3.41404907, 3.74451919,\n",
      "       4.11921628, 3.32409405, 0.29615581, 3.64243551]), 'param_colsample_bytree': masked_array(data=[0.4, 0.4, 0.4, 0.6, 0.6, 0.6, 0.8, 0.8, 0.8],\n",
      "             mask=[False, False, False, False, False, False, False, False,\n",
      "                   False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_subsample': masked_array(data=[0.4, 0.6, 0.8, 0.4, 0.6, 0.8, 0.4, 0.6, 0.8],\n",
      "             mask=[False, False, False, False, False, False, False, False,\n",
      "                   False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'colsample_bytree': 0.4, 'subsample': 0.4}, {'colsample_bytree': 0.4, 'subsample': 0.6}, {'colsample_bytree': 0.4, 'subsample': 0.8}, {'colsample_bytree': 0.6, 'subsample': 0.4}, {'colsample_bytree': 0.6, 'subsample': 0.6}, {'colsample_bytree': 0.6, 'subsample': 0.8}, {'colsample_bytree': 0.8, 'subsample': 0.4}, {'colsample_bytree': 0.8, 'subsample': 0.6}, {'colsample_bytree': 0.8, 'subsample': 0.8}], 'split0_test_score': array([-0.40017132, -0.40006799, -0.39997053, -0.40003389, -0.39983567,\n",
      "       -0.3997434 , -0.40001164, -0.3997918 , -0.39979197]), 'split1_test_score': array([-0.40035512, -0.40018436, -0.40016729, -0.40011153, -0.39987701,\n",
      "       -0.3998872 , -0.40002578, -0.39981984, -0.39986001]), 'split2_test_score': array([-0.40019261, -0.40004518, -0.40001364, -0.39996625, -0.39973484,\n",
      "       -0.39970693, -0.39994091, -0.39973835, -0.3996646 ]), 'mean_test_score': array([-0.40023968, -0.40009918, -0.40005049, -0.40003722, -0.39981584,\n",
      "       -0.39977918, -0.39999278, -0.39978333, -0.39977219]), 'std_test_score': array([8.20867918e-05, 6.09484069e-05, 8.44487777e-05, 5.93596111e-05,\n",
      "       5.97105096e-05, 7.78241902e-05, 3.71274173e-05, 3.38040929e-05,\n",
      "       8.09930858e-05]), 'rank_test_score': array([9, 8, 7, 6, 4, 2, 5, 3, 1]), 'split0_train_score': array([-0.39844552, -0.39832006, -0.39826169, -0.39794808, -0.39782063,\n",
      "       -0.39774505, -0.3977425 , -0.39756161, -0.39758671]), 'split1_train_score': array([-0.39842499, -0.39829107, -0.39826251, -0.39797166, -0.39767065,\n",
      "       -0.39769998, -0.39765537, -0.39745669, -0.39750719]), 'split2_train_score': array([-0.39850924, -0.39837909, -0.39834831, -0.39796188, -0.39778401,\n",
      "       -0.39774641, -0.39772671, -0.39759234, -0.39751002]), 'mean_train_score': array([-0.39845991, -0.39833008, -0.39829084, -0.39796054, -0.39775843,\n",
      "       -0.39773048, -0.39770819, -0.39753688, -0.39753464]), 'std_train_score': array([3.58686264e-05, 3.66239406e-05, 4.06453495e-05, 9.67402442e-06,\n",
      "       6.38438229e-05, 2.15735691e-05, 3.79064452e-05, 5.80746375e-05,\n",
      "       3.68397644e-05])}\n",
      "-0.39977219393745533\n",
      "logloss of train : 0.3980412782525945\n",
      "logloss of test : 0.4033660529015553\n"
     ]
    }
   ],
   "source": [
    "print(gsearch2_2.cv_results_)\n",
    "print(gsearch2_2.best_score_)\n",
    "y_pred_train = gsearch2_2.predict_proba(X_train)\n",
    "logloss = log_loss(y_train, y_pred_train)\n",
    "print (\"logloss of train :\",logloss )\n",
    "\n",
    "y_pred_test = gsearch2_2.predict_proba(X_test)\n",
    "logloss = log_loss(y_test, y_pred_test)\n",
    "print (\"logloss of test :\",logloss )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.399807 using {'min_child_samples': 20, 'min_child_weight': 2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEKCAYAAAAvlUMdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xu8F3W97/HXm8UCBCGNi4qQIIIJssRcsTXNyKRQj6BlJt6pI+GBMNuWunN7O1mkOzJPBLkLKC2vqbFPuvGyNUNwy0IXIChy0RTQwxIlRRHW5XP+mFmL31qsO2v4gbyfj8fvwcx3vt+Zz4zL+fy+M/ObryICMzOzttYu3wGYmdnHkxOMmZllwgnGzMwy4QRjZmaZcIIxM7NMOMGYmVkmnGDMzCwTTjBmZpYJJxgzM8tE+3wHkE89evSIfv365TsMM7M9yqJFi96OiJ5N1durE0y/fv0oKSnJdxhmZnsUSX9vTj1fIjMzs0w4wZiZWSacYMzMLBNOMGZmlgknGDMzy4QTjJmZZcIJxszMMrFX/w6mtX763E95+Z2X8x2GmVmrffqTn+bK4Vdmug33YMzMLBPuwbRC1lnfzOzjwD0YMzPLhBOMmZllwgnGzMwy4QRjZmaZcIIxM7NMOMGYmVkmnGDMzCwTTjBmZpYJJxgzM8uEE4yZmWUi0wQjaZSkFZJWSbqqnuUTJC2VVCppnqTBaXkHSbPSZYsljchpMzYtXyLpPyX1yFn2nXR7yyTdnOW+mZlZ4zJLMJIKgGnAKcBgYGx1Asnxx4gYGhHDgJuBqWn5JQARMRQYCfxMUjtJ7YFfAF+MiCJgCTAp3d4XgTFAUUQMAf4tq30zM7OmZdmDGQ6siog1EbENuJskAdSIiPdyZrsAkU4PBp5I62wANgHFgNJPF0kCugHr0zaXAlMiYmtOOzMzy5MsE8zBwBs582vTslokTZS0mqQHMzktXgyMkdReUn/gGKBvRJSTJJKlJIllMPDbtM0g4POS/lvSXyV9NoudMjOz5skywaiestihIGJaRAwArgSuSYtnkiSkEuBWYD5QIamQJMEcDfQmuUR2ddqmPbA/cCzwfeDetJdTOyhpvKQSSSVlZWU7sXtmZtaYLBPMWqBvznwftl/Oqs/dwBkAEVEREZdHxLCIGAPsB6wEhqXLV0dEAPcCn8vZ3gOReA6oAnrU3UhE3B4RxRFR3LNnz53bQzMza1CWCWYhMFBSf0kdgHOAObkVJA3MmT2NJIkgqbOkLun0SKAiIpYD64DBkqozw0jgpXT6IeCktM0goAPwdhY7ZmZmTctsRMuIqJA0CZgLFAAzI2KZpBuBkoiYA0ySdDJQDrwLXJQ27wXMlVRFklQuSNe5XtINwNOSyoG/AxenbWYCMyW9CGwDLkp7OWZmlgfam8/BxcXFUVJSku8wzMz2KJIWRURxU/X8S34zM8uEE4yZmWXCCcbMzDLhBGNmZplwgjEzs0w4wZiZWSacYMzMLBNOMGZmlgknGDMzy4QTjJmZZcIJxszMMuEEY2ZmmXCCMTOzTDjBmJlZJpxgzMwsE04wZmaWCScYMzPLhBOMmZllwgnGzMwy4QRjZmaZcIIxM7NMOMGYmVkmMk0wkkZJWiFplaSr6lk+QdJSSaWS5kkanJZ3kDQrXbZY0oicNmPT8iWS/lNSjzrrvEJS1C03M7NdK7MEI6kAmAacAgwGxlYnkBx/jIihETEMuBmYmpZfAhARQ4GRwM8ktZPUHvgF8MWIKAKWAJNyttk3rf96VvtlZmbNk2UPZjiwKiLWRMQ24G5gTG6FiHgvZ7YLEOn0YOCJtM4GYBNQDCj9dJEkoBuwPmcdPwd+kLMeMzPLkywTzMHAGznza9OyWiRNlLSapAczOS1eDIyR1F5Sf+AYoG9ElAOXAktJEstg4LfpekYD6yJicUb7Y2ZmLZBlglE9ZTv0LCJiWkQMAK4ErkmLZ5IkpBLgVmA+UCGpkCTBHA30JrlEdrWkzsAPgWubDEoaL6lEUklZWVnL98rMzJolywSzFuibM9+H2pez6robOAMgIioi4vKIGBYRY4D9gJXAsHT56ogI4F7gc8AAoD+wWNJr6bael3Rg3Y1ExO0RURwRxT179tzZfTQzswZkmWAWAgMl9ZfUATgHmJNbQdLAnNnTSJIIkjpL6pJOjwQqImI5sA4YLKk6M4wEXoqIpRHRKyL6RUQ/kuT2mYh4K8P9MzOzRrTPasURUSFpEjAXKABmRsQySTcCJRExB5gk6WSgHHgXuCht3guYK6mKJKlckK5zvaQbgKcllQN/By7Oah/MzKz1lFxp2jsVFxdHSUlJvsMwM9ujSFoUEcVN1fMv+c3MLBNOMGZmlgknGDMzy4QTjJmZZcIJxszMMuEEY2ZmmXCCMTOzTDjBmJlZJpxgzMwsE04wZmaWCScYMzPLhBOMmZllwgnGzMwy4QRjZmaZcIIxM7NMOMGYmVkmnGDMzCwTTjBmZpYJJxgzM8uEE4yZmWXCCcbMzDLhBGNmZplon+XKJY0CfgEUAL+JiCl1lk8AJgKVwGZgfEQsl9QB+DVQDFQBl0XEU2mbscC/AAGsB86PiLcl3QKcDmwDVgPjImJTlvtn1hrl5eWsXbuWjz76KN+hmDWqU6dO9OnTh8LCwla1V0S0cUjpiqUC4BVgJLAWWAiMjYjlOXW6RcR76fRo4H9FxChJE4HiiBgnqRfwCPBZkh7XemBwmlRuBj6MiOslfRn4r4iokPRTgIi4srEYi4uLo6SkpK133axRr776Kl27dqV79+5Iync4ZvWKCDZu3Mj7779P//79ay2TtCgiiptaR5aXyIYDqyJiTURsA+4GxuRWqE4uqS4kvRKAwcATaZ0NwCaS3ozSTxcl/2d2I0k4RMSjEVGRtn8W6JPFTpntrI8++sjJxXZ7kujevftO9bSzTDAHA2/kzK9Ny2qRNFHSauBmYHJavBgYI6m9pP7AMUDfiCgHLgWWkvZkgN/Ws+1vkvR6zHZLTi62J9jZv9MsE0x9ke1wPS4ipkXEAOBK4Jq0eCZJQioBbgXmAxWSCkkSzNFAb2AJcHWtjUo/BCqAP9QblDReUomkkrKystbsl5mZNUOTCUbSAEkd0+kRkiZL2q8Z614L9M2Z70N6OasBdwNnAERERURcHhHDImIMsB+wEhiWLl8dyc2je4HP5cR6EfA/gPOigZtLEXF7RBRHRHHPnj2bsRtmHz+bNm3iV7/6VYvbnXrqqWzalO2zM6+99hpHHnlkpttoyL777tviNq2J98c//nGLt9NcEcHkyZM57LDDKCoq4vnnn2+0/ujRozM73s3pwfwJqJR0GMnlqP7AH5vRbiEwUFL/9Kmwc4A5uRUkDcyZPY0kiSCps6Qu6fRIoCJ9OGAdMFhSdWYYCbyU1htF0gsaHREfNiM+s71WQwmmsrKy0XYPP/ww++3XnO+X1piGEkxEUFVVtVPrfuSRR1i5ciUrV67k9ttv59JLL22w7gMPPNCqpNpczXlMuSp9MutM4NaI+D+SXmiqUdpmEjCX5DHlmRGxTNKNQElEzAEmSToZKAfeBS5Km/cC5kqqIkkqF6TrXC/pBuBpSeXA34GL0za/BDoCj6XXDZ+NiAnN2D+zvLnhP5axfP17TVdsgcG9u3Hd6UMarXPVVVexevVqhg0bRmFhIfvuuy8HHXQQpaWlLF++nDPOOIM33niDjz76iMsuu4zx48cD0K9fP0pKSti8eTOnnHIKJ5xwAvPnz+fggw/mz3/+M/vss0+921u1ahUTJkygrKyMgoIC7rvvPg499FB+8IMf8MgjjyCJa665hm984xu12i1btoxx48axbds2qqqq+NOf/sTAgQMbjG/fffdl4sSJPP744+y///78+Mc/5gc/+AGvv/46t956K6NHj2b27Nk8+OCDbN26lVdffZVzzz2X6667boeYb7nlFu699162bt3KmWeeyQ033NDg8ayoqOCiiy7ihRdeYNCgQfz+979nwYIF/PKXv+TBBx8E4LHHHmP69OkMGjSILVu2MGzYMIYMGcJNN93EKaecwhe/+EUWLFjAQw89xIoVK7juuuvYunUrAwYMYNasWey7774sWrSI733ve2zevJkePXowe/ZsDjrooFqx/PnPf+bCCy9EEsceeyybNm3izTff3KHe5s2bmTp1Krfffjtnn312o38vrRYRjX6A/wbGAi8C/dOyF5tqtyd8jjnmmDDb1ZYvX14zff2cF+PsGfPb9HP9nBebjOHVV1+NIUOGRETEk08+GZ07d441a9bULN+4cWNERHz44YcxZMiQePvttyMi4pBDDomysrJ49dVXo6CgIF544YWIiPj6178ed9xxR4PbGz58eDzwwAMREbFly5b44IMP4v7774+TTz45Kioq4q233oq+ffvG+vXra8U2adKkuPPOOyMiYuvWrfHhhx82Gh8QDz/8cEREnHHGGTFy5MjYtm1blJaWxlFHHRUREbNmzYoDDzww3n777Zr2CxcujIiILl26RETE3Llz45JLLomqqqqorKyM0047Lf761782eCyBmDdvXkREjBs3Lm655ZaoqqqKww8/PDZs2BAREWPHjo05c+bU2k51e0mxYMGCiIgoKyuLz3/+87F58+aIiJgyZUrccMMNsW3btjjuuONq1nf33XfHuHHjIiJi+vTpMX369IiIOO200+Jvf/tbzfpPOumkmv3L9d3vfjceeOCBWse7Prl/r9VIOglNnmOb04MZB0wAboqIV9Onuu7MItmZ7W2a6mnsKsOHD6/1W4fbbrut5pv3G2+8wcqVK+nevXutNv3792fYsGEAHHPMMbz22mv1rvv9999n3bp1nHnmmUDy4z2AefPmMXbsWAoKCjjggAP4whe+wMKFCykqKqppe9xxx3HTTTexdu1avvrVrzJw4MBG4+vQoQOjRo0CYOjQoXTs2JHCwkKGDh1aK76RI0fW7M9Xv/pV5s2bR3Hx9p91PProozz66KMcffTRQPJtf+XKlZx44on17mPfvn05/vjjATj//PO57bbbuOKKK7jgggu48847GTduHAsWLOD3v/99ve0POeQQjj32WACeffZZli9fXrO+bdu2cdxxx7FixQpefPFFRo4cCSSXM6t7JRMmbL9YE/Xcfq77NFhpaSmrVq3i5z//eYP/3dpCkwkmknsfkwEk7Q90jTq/yDezPVuXLl1qpp966ikef/xxFixYQOfOnRkxYkS9v4Xo2LFjzXRBQQFbtmypd931nfAaK8917rnn8k//9E/85S9/4Stf+Qq/+c1vaNeuXYPxFRYW1pxM27VrVxNju3btqKioqFlv3RNu3fmI4Oqrr+bb3/52kzE2tr5x48Zx+umn06lTJ77+9a/Tvn39p9zc4x8RjBw5krvuuqtWnaVLlzJkyBAWLFjQaCx9+vThjTe2/0Jk7dq19O7du1adBQsWsGjRIvr160dFRQUbNmxgxIgRPPXUU03ua0s05ymypyR1k/RJkt+nzJI0tU2jMLNdqmvXrrz//vv1LvvHP/7B/vvvT+fOnXn55Zd59tlnd2pb3bp1o0+fPjz00EMAbN26lQ8//JATTzyRe+65h8rKSsrKynj66acZPnx4rbZr1qzh0EMPZfLkyYwePZolS5a0SXyPPfYY77zzDlu2bOGhhx6q6S1U+8pXvsLMmTPZvHkzAOvWrWPDhg0Nru/111+vOfHfddddnHDCCQD07t2b3r1786Mf/YiLL764pn5hYSHl5eX1ruvYY4/lmWeeYdWqVQB8+OGHvPLKKxx++OGUlZXVbKe8vJxly5bt0H706NH8/ve/JyJ49tln+cQnPrHD/ZdLL72U9evX89prrzFv3jwGDRrU5skFmvcU2Sci+cX9V4FZEXEMcHKbR2Jmu0z37t05/vjjOfLII/n+979fa9moUaOoqKigqKiIf/3Xf625dLMz7rjjDm677TaKior43Oc+x1tvvcWZZ55JUVERRx11FCeddBI333wzBx54YK1299xzD0ceeSTDhg3j5Zdf5sILL2yT+E444QQuuOAChg0bxte+9rVal8cAvvzlL3Puuedy3HHHMXToUM4666wGEzLAEUccwe9+9zuKiop45513aj25dd5559G3b18GDx5cUzZ+/HiKioo477zzdlhXz549mT17NmPHjqWoqIhjjz2Wl19+mQ4dOnD//fdz5ZVXctRRRzFs2DDmz58PwIwZM5gxYwaQPEp+6KGHcthhh3HJJZfUelqw+pLmrtLku8gkLQW+DPwO+GFELJS0JCKKGm24B/C7yCwfXnrpJY444oh8h7HXmj17NiUlJfzyl7/cJdubNGkSRx99NN/61rd2yfbaWn1/r819F1lzbvLfSPKo8TNpcjmU9PcqZmbWsGOOOYYuXbrws5/9LN+h5EVmb1PeE7gHY/nwce7BTJw4kWeeeaZW2WWXXca4cePyFFHb2bhxI1/60pd2KH/iiSd2eMLu4yTTHoykPsD/AY4neZfYPJLxWda2Llwz+7iaNm1avkPITPfu3SktLc13GHuU5tzkn0XyipfeJG9D/o+0zMzMrEHNSTA9I2JWJC+grIiI2YDfEmlmZo1qToJ5W9L5kgrSz/nAxqwDMzOzPVtzEsw3gbOBt4A3gbNIXh9jZmbWoCYTTES8HhGjI6JnRPSKiDNIfnRpZnsojwdTv71pPJht27Yxfvx4Bg0axKc//Wn+9Kc/tXksrR3R8nttGoWZ7VIeDya/dofxYG666SZ69erFK6+8wvLly/nCF76wU9utT3N+aFkfDyhu1hYeuQreWtq26zxwKJzS+PtoPR6Mx4OZOXMmL7/8MpC8DLRHjx6N/cm0TnPe6V/3A7zemna728fjwVg+1Bpf4+ErI2ae2rafh69sMgaPB7N3jwfz7rvvRp8+feLyyy+Po48+Os4666x466236t2/TMaDkfQ+yQ8rd1gE1P81xcxapomexq7i8WD2rvFgKioqWLt2LccffzxTp05l6tSpXHHFFdxxxx31xtdaDSaYiOjaplsys92Wx4PZu8aD6d69O507d65J+l//+tf57W9/28Retlxrb/Kb2R7M48Hs3ePBSOL000+vGQPmiSeeqDWcQFtxgjHbC3k8GI8H89Of/pTrr7+eoqIi7rjjjkze+Oy3KfttyraLfZzfprwn8HgwLZP1eDBmZtYKe/t4MM15XX99T5P9AygB/jki1jTSdhTwC6AA+E1ETKmzfAIwEagENgPjI2K5pA7Ar4FioIpkeICn0jZjgX9JY1oPnB8Rb0v6JHAP0A94DTg7It5tav/MrO3sCePBXHzxxbXuhzRXa8aDWbRoUYu383HSnCGTbyA5kf+R5BHlc4ADgRXApRExooF2BcArwEhgLbAQGBsRy3PqdIuI99Lp0cD/iohRkiYCxRExTlIv4BHgsyT3jNYDg9OkcjPwYURcn06/ExFTJF0F7B8RVza2b75EZvngS2S2J9mZS2TNuck/KiJ+HRHvR8R7EXE7cGpE3APs30i74cCqiFgTEduAu4ExuRWqk0uqC9t7SoOBJ9I6G4BNJL0ZpZ8uSp4D7EaScEjX/bt0+nfAGc3YNzMzy0hzEkyVpLMltUs/Z+csa6z7czDwRs782rSsFkkTJa0GbgYmp8WLgTGS2kvqDxwD9I2IcuBSYClpTwaofnj7gIh4EyD9t1cz9s3MzDLSnARzHnABsCH9XACcL2kfYFIj7ep7X9kOCSkipkXEAOBK4Jq0eCZJQioBbgXmAxWSCkkSzNEkI2wuAa5uxj5sD0oaL6lEUklZWVlLmpqZWQs0eZM/vYl/egOL5zXSdC3QN2e+D9svZ9XnbmB6us0K4PLqBZLmAyuBYeny1Wn5vcBVabX/J+mgiHhT0kEkybC+/bkduB2SezCNxGNmZjuhyR6MpD6SHpS0QdL/k/QnSX2ase6FwEBJ/dOnws4B5tRZ98Cc2dNIkgiSOkvqkk6PBCrShwPWAYMlVQ/ZPBJ4KZ2eA1yUTl8E/LkZMZrtlTweTP32pvFg7rrrLoYOHUpRURGjRo3i7bffbvNYmnOJbBbJybs3yT2U/0jLGpX2QiYBc0mSwL0RsUzSjekTYwCTJC2TVEoyxkx1gugFPC/pJZJLZxek61wP3AA8LWkJSY+m+r/UFGCkpJUkiWf3eIug2W7I48HkV77Hg6moqOCyyy7jySefZMmSJRQVFWXyw9Pm/NCyZ0TkJpTZkr7bnJVHxMPAw3XKrs2ZvqyBdq8BhzewbAYwo57yjcCOD6mb7cZ++txPefmdl9t0nZ/+5Ke5cnijT+h7PJi9fDyY6tfpf/DBB3Tv3p333nuPww47rNG/mdZoTg/mbUnnSypIP+cDG9s8EjPbZaZMmcKAAQMoLS3llltu4bnnnuOmm25i+fLkZ2ozZ85k0aJFlJSUcNttt7Fx447/y69cuZKJEyeybNky9ttvv0aH3D3vvPOYOHEiixcvZv78+Rx00EE88MADlJaWsnjxYh5//HG+//3v8+abb9ZqN2PGDC677DJKS0spKSmhT58+jcb3wQcfMGLECBYtWkTXrl255ppreOyxx3jwwQe59tqa77Y899xz/OEPf6C0tJT77ruPur+He/TRR1m5ciXPPfccpaWlLFq0iKeffrrB/VuxYgXjx49nyZIldOvWjV/96lecdNJJvPTSS1Q/TDRr1izGjRvHlClT2GeffSgtLeUPf/hDTfsLL7yQF154gS5duvCjH/2Ixx9/nOeff57i4mKmTp1KeXk53/nOd7j//vtZtGgR3/zmN/nhD39Yc5yq30W2bt06+vbdfvu7T58+rFu3rla8hYWFTJ8+naFDh9K7d2+WL1+eyatsmtOD+SbwS+DnJE+BzQd2n5/lmu3Bmupp7CoeD2bvGg+mvLyc6dOn88ILL3DooYfyne98h5/85Cdcc801O7TdGc15iux1YHRuWXqJ7NY2jcTM8sbjwexd48GUlpYCMGDAAADOPvtspkxp+9vWrX1d//faNAoz26U8HszePR7MwQcfzPLly2su3z322GOZvL6otW9Tru9HlGa2h8gdD2afffbhgAMOqFk2atQoZsyYQVFREYcffnibjQfz7W9/m2uvvZbCwkLuu+8+zjzzTBYsWMBRRx2FpJrxYHIvZd1zzz3ceeedFBYWcuCBB3LttdfSpUuXnY6vejyYVatWce6559Y7HsxLL73EcccdByQPD9x555306lX/C0Kqx4P59re/zcCBA3cYD6asrKze8WA+85nPcNNNN9VaV+54MFu3bgXgRz/6EYMGDeL+++9n8uTJ/OMf/6CiooLvfve7DBkypOb+y4QJEzj11FN5+OGHOeyww+jcuTOzZm1/RmvYsGGUlpbSu3dvrrvuOk488UQKCws55JBDmD17douPY1NaNR6MpNcj4lNtHs0u5pddWj74ZZf55fFgWiaT8WAaeE0/JL2X+p9FNDOzGh4PpgER0XVXBmJmez6PB1Pb3j4ejEe0NMuDiNjhyaOPg2nTpuU7hMx079695umrvUVrbqHkau1TZGbWSp06dWLjxo07/T+vWZYigo0bN9b8bqk13IMx28X69OnD2rVr8XARtrvr1KlTzdsTWsMJxmwXKywsrPWrebOPK18iMzOzTDjBmJlZJpxgzMwsE04wZmaWCScYMzPLhBOMmZllwgnGzMwy4QRjZmaZcIIxM7NMOMGYmVkmMk0wkkZJWiFplaSr6lk+QdJSSaWS5kkanJZ3kDQrXbZY0oi0vGtat/rztqRb02WfkvSkpBckLZF0apb7ZmZmjcvsXWSSCoBpwEhgLbBQ0pyIWJ5T7Y8RMSOtPxqYCowCLgGIiKGSegGPSPpsRLwPDMvZxiLggXT2GuDeiJieJqqHgX5Z7Z+ZmTUuyx7McGBVRKyJiG3A3cCY3AoR8V7ObBe2j6A5GHgirbMB2ATUGp5T0kCgF/C36tUB3dLpTwDr22xPzMysxbJMMAcDb+TMr03LapE0UdJq4GZgclq8GBgjqb2k/sAxQN86TccC98T2QTWuB86XtJak9/KdttoRMzNruSwTTH3D9e0wwlJETIuIAcCVJJe5AGaSJKQS4FZgPlBRp+k5wF0582OB2RHRBzgVuEPSDvsnabykEkklHo/DzCw7WSaYtdTudfSh8ctWdwNnAERERURcHhHDImIMsB+wsrqipKOA9hGRO+D1t4B70/YLgE5Aj7obiYjbI6I4Iop79uzZuj0zM7MmZZlgFgIDJfWX1IGkxzEnt0J6H6XaaaRJRFJnSV3S6ZFARZ2HA8ZSu/cC8DrwpbTNESQJxl0UM7M8yewpsoiokDQJmAsUADMjYpmkG4GSiJgDTJJ0MlAOvAtclDbvBcyVVAWsAy6os/qzSS6D5fpn4N8lXU5yKe7i8KDnZmZ5o735HFxcXBwlJSX5DsPMbI8iaVFEFDdVz7/kNzOzTDjBmJlZJpxgzMwsE04wZmaWCScYMzPLhBOMmZllwgnGzMwy4QRjZmaZcIIxM7NMOMGYmVkmnGDMzCwTTjBmZpYJJxgzM8tEZq/r/1h78QF4/nf5jsLMaqlvEF1r0D9NgMNHZboJJ5jWqKqA8i35jsLMqu3Fw460WlV55ptwgmmNorOTj5mZNcj3YMzMLBNOMGZmlgknGDMzy4QTjJmZZcIJxszMMuEEY2ZmmXCCMTOzTGSaYCSNkrRC0ipJV9WzfIKkpZJKJc2TNDgt7yBpVrpssaQRaXnXtG71521Jt+as72xJyyUtk/THLPfNzMwal9kPLSUVANOAkcBaYKGkORGxPKfaHyNiRlp/NDAVGAVcAhARQyX1Ah6R9NmIeB8YlrONRcAD6fRA4Grg+Ih4N21nZmZ5kmUPZjiwKiLWRMQ24G5gTG6FiHgvZ7YLUP2+h8HAE2mdDcAmoDi3bZpQegF/S4suAaZFxLs57czMLE+yTDAHA2/kzK9Ny2qRNFHSauBmYHJavBgYI6m9pP7AMUDfOk3HAvdE1LyEaBAwSNIzkp6VVO9b3CSNl1QiqaSsrKzVO2dmZo3LMsHU92rTHd5IFxHTImIAcCVwTVo8kyQhlQC3AvOBijpNzwHuyplvDwwERpAkn99I2q+e7d0eEcURUdyzZ88W7ZCZmTXOQ1Q0AAANJ0lEQVRfli+7XEvtXkcfYH0j9e8GpgNERAVwefUCSfOBlTnzRwHtI2JRne09GxHlwKuSVpAknIU7uR9mZtYKWfZgFgIDJfWX1IGkxzEnt0J6H6XaaaRJRFJnSV3S6ZFARZ2HA8ZSu/cC8BDwxbRND5JLZmvabnfMzKwlMuvBRESFpEnAXKAAmBkRyyTdCJRExBxgkqSTgXLgXeCitHkvYK6kKmAdcEGd1Z8NnFqnbC7wZUnLgUrg+xGxMYt9MzOzpin24oF6iouLo6SkJN9hmJntUSQtiojipur5l/xmZpYJJxgzM8uEE4yZmWXCCcbMzDLhBGNmZplwgjEzs0w4wZiZWSacYMzMLBNOMGZmlgknGDMzy4QTjJmZZcIJxszMMuEEY2ZmmXCCMTOzTGQ5oqWZNaGqKvioopKPyqv4qLwy/VSxrbKK9u1Eh/bt6FDQjsL27SgsEB0LCihsLzoUtKOgnZDqG5ncbPfgBGOWo7yyquYk/1F5JVtrnfzTf+skhK0VtZNDUqeKLduq29ffdmuaSFpLgg4F2xNQ8m+SfAoL2m1PTul08q/qKatbT420LahZXqtt+m9uDE6A5gRju62IYFtlFR+VV7G1+iSdnrC3bEtO4jUn+ZxltRJCRc7yepLDR+VVbK3Yvr7KqtYNwCdBp/YFdCpsR6fCAjoVFtCxfTK9T2EBPfZtX1PeqbAdHdtvn+5UWECn9u1qLe/Qvh0Vlcn+l1dWUV4RbK2sorwiSUrlFUn51nTZtspKyisip6yqVttNW8rrlCXT22rKotX73tgxcQLcuznBWLNVVcX2b+v1nKg/qqjcngiqy9Nv8smyHevX7i3suLy1A662b6c6J/PaJ/D99ilMkkDNCX57nX2q21XXr3Pyz63fsTohFLTb409WlVVJgqpOPOW1/t2e7LblJLnaZVErAW6rrJ7engC3l9Vu++GWyh3Kyp0A9/i/KSeYPVhFZVWtb/HNvayztbyy3na1y3ZMFNsqWn85p0NBu+0n88J26Uk8OVF37dSenl071jmZ79gTqCmv++2/7nT7drQv8PMrLVXQThS0S47h7ijfCbB6W7nb3VpTtuclwGMP7c6gA7q2acx1OcG0kYigvDJ2vCTTwGWdrQ2c/OvtCdRZvjUtK69s/R903W/iybf55OS8f+cOOy7vUFDPiX37yb6+nkBuD6Kg3Z79Tczyb09JgFsbTERRKwHWTVjbKmN7WU4C3F4WO5btRAK86cwjnWB2R/csfJ1f/3XNDt/6W/sFpp3IuTSTnqxzTtTd9imsOZl3zD3Bt69zsi8sqLkcVNNDqFO/Y2E7Orbf8y/nmO1u9rQE2LlD9nE6wbTCJ7t05Ije3ejUvoB9Oux4yaa6J9DQdfu63/gLC/b8a61mtnvLRwLMNMFIGgX8AigAfhMRU+osnwBMBCqBzcD4iFguqQPwa6AYqAIui4inJHUF/pazij7AnRHx3Zx1ngXcB3w2Ikqy2K+Rgw9g5OADsli1mdnHRmYJRlIBMA0YCawFFkqaExHLc6r9MSJmpPVHA1OBUcAlABExVFIv4BFJn42I94FhOdtYBDyQM98VmAz8d1b7ZWZmzZPlozbDgVURsSYitgF3A2NyK0TEezmzXYDquxiDgSfSOhuATSS9mRqSBgK9qN2j+d/AzcBHbbcbZmbWGlkmmIOBN3Lm16ZltUiaKGk1SWKYnBYvBsZIai+pP3AM0LdO07HAPRHJLyUkHQ30jYj/21hQksZLKpFUUlZW1pr9MjOzZsgywdR313qH56wiYlpEDACuBK5Ji2eSJKQS4FZgPlBRp+k5wF0AktoBPwf+uamgIuL2iCiOiOKePXs2c1fMzKylsrzJv5bavY4+wPpG6t8NTAeIiArg8uoFkuYDK3PmjwLaR8SitKgrcCTwVPo01oHAHEmjs7rRb2ZmjcuyB7MQGCipf/pU2DnAnNwK6X2UaqeRJhFJnSV1SadHAhV1Hg4YS9p7AYiIf0REj4joFxH9gGcBJxczszzKrAcTERWSJgFzSR5TnhkRyyTdCJRExBxgkqSTgXLgXeCitHkvYK6kKmAdcEGd1Z8NnJpV7GZmtvMUrX2b4MdAcXFxlJS4k2Nm1hKSFkVEcZP19uYEI6kM+Hsrm/cA3m7DcNqK42oZx9Vyu2tsjqtldiauQyKiyaek9uoEszMklTQng+9qjqtlHFfL7a6xOa6W2RVx+Z3mZmaWCScYMzPLhBNM692e7wAa4LhaxnG13O4am+Nqmczj8j0YMzPLhHswZmaWCSeYekgaJWmFpFWSrmqk3lmSQlJxTtnVabsVkr6yO8QlqZ+kLZJK08+MXRmXpIslleVs/3/mLLtI0sr0c1HdtnmMqzKnfE7dtlnGldY5W9JyScsk/TGnPG/Hq4m48na8JP08Z9uvSNqUsyyff1+NxZXP4/UpSU9KekHSEkmn5ixr2/NXRPiT8yF568Bq4FCgA8mbnQfXU68r8DTJa2mK07LBaf2OQP90PQW7QVz9gBfzdbyAi4Ff1tP2k8Ca9N/90+n98x1XumxzHo/XQOCF6mMB9NpNjle9ceX7eNWp/x2St4bk/Xg1FFe+jxfJvZdL0+nBwGs50216/nIPZkdNjmOTqm/smTHA3RGxNSJeBVal68t3XFlqblz1+QrwWES8ExHvAo+RDDiX77iy1Jy4LgGmpceESMZEgvwfr4biylJL/zvmvqcw38erobiy1Jy4AuiWTn+C7S8hbvPzlxPMjpocx0YNjz3TrDFw8hAXQP+0S/xXSZ9vo5iaFVfqa2l3/H5J1W/ZzuvxaiQugE5Kxg16VtIZbRRTc+MaBAyS9Ey6/VEtaJuPuCC/xwsASYeQfPP+r5a23cVxQX6P1/XA+ZLWAg+T9K6a27ZFsnxd/56q0XFstH3smYtb2nYn7UxcbwKfioiNko4BHpI0JGqPKJpJXKn/AO6KiK2SJgC/A05qZtt8xAXJ8Vov6VDgvyQtjYjVuyiu9iSXo0aQDHPxN0lHNrPtLo8rIjaR3+NV7Rzg/oiobEXbltqZuCC/x2ssMDsifibpOOCOrP6+3IPZUVPj2OSOPfMacCzJ2DPFzWibl7jSLu9GgEjG0FlN8m10V8RFRGyMiK3p7L+TjFDarLZ5iouIWJ/+uwZ4Cjh6V8WV1vlzRJSnlypWkJzY83q8Gokr38erWs0ghK1ouyvjyvfx+hZwb7r9BUAnkveStf3xyuJG0578IfmWtoakS1t9k2xII/WfYvvN9CHUvkm2hra7yb8zcfWsjoPk5t864JO7Ki7goJzpM4Fn0+lPAq+S3IDdP53eHeLaH+iYTvcgGaeowRu4GcQ1CvhdzvbfALrvBserobjyerzSeocDr5H+tm93+PtqJK58/309AlycTh9BkkREBuevnd6hj+OHZKyZV0i+6f8wLbuRZBCzunWfIj2Rp/M/TNutAE7ZHeICvgYsS/94ngdO35VxAT/J2f6TwKdz2n6T5GbiKmDc7hAX8DlgaVq+FPjWLo5LwFRgebr9c3aT41VvXPk+Xun89cCUetrm7Xg1FFe+jxfJ02LPpNsvBb6c07ZNz1/+Jb+ZmWXC92DMzCwTTjBmZpYJJxgzM8uEE4yZmWXCCcbMzDLhBGOWEUnXS7piN4jjNUk98h2H7X2cYMzMLBNOMGYtIKmLpL9IWizpRUnfyO0hSCqW9FROk6Mk/Vc6HsklaZ2DJD2djgXyYvXLRyVNT1+AuEzSDTnbfE3SjyUtSJd/RtJcSavTd6ghaUS6zgeVjNcyI30/Xd34z5f0XLrtX0sqyPJ42d7NCcasZUYB6yPiqIg4EvjPJuoXAacBxwHXSuoNnAvMjYhhwFEkv6aG5FfXxWmbL0gqylnPGxFxHPA3YDZwFsn75m7MqTMc+GdgKDAA+GpuIJKOAL4BHJ9uuxI4rwX7btYifpuyWcssBf5N0k+B/xsRf5PqewltjT9HxBZgi6QnSZLAQmCmpELgoYioTjBnSxpP8v/lQSSv9FiSLqse9XApsG9EvA+8L+kjSfuly56L5OWJSLoLOAG4PyeWL5G80HNhGvM+wK4Y08X2Uk4wZi0QEa+kQx6cCvxE0qNABduvBnSq22THVcTTkk4k6dncIekWkp7JFcBnI+JdSbPrrKv6rc9VOdPV89X/H++wrTrzInlZ5dVN7KZZm/AlMrMWSC9xfRgRdwL/BnyG5G251a/6/1qdJmMkdZLUnWQclYXpAFQbIuLfgd+m6+gGfAD8Q9IBwCmtCG+4pP7pvZdvAPPqLH8COEtSr3RfPpnGYpYJ92DMWmYocIukKqAcuJTkUtNvJf0L8N916j8H/AX4FPC/Ixlk6iLg+5LKgc3AhRHxqqQXSN7uvIbkbbcttQCYksb4NPBg7sKIWC7pGuDRNAmVAxOBv7diW2ZN8tuUzT4GJI0AroiI/5HvWMyq+RKZmZllwj0YMzPLhHswZmaWCScYMzPLhBOMmZllwgnGzMwy4QRjZmaZcIIxM7NM/H+uGKmVujZqjgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2a69ca842b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_2.best_score_, gsearch2_2.best_params_))\n",
    "test_means = gsearch2_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "pd.DataFrame(gsearch2_2.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(subsample), len(colsample_bytree))\n",
    "train_scores = np.array(train_means).reshape(len(subsample), len(colsample_bytree))\n",
    "\n",
    "for i, value in enumerate(subsample):\n",
    "#    pyplot.plot(colsample_bytree, -test_scores[i], label= 'valid_colsample_bytree:'   + str(value))\n",
    "    pyplot.plot(subsample, -train_scores[i], label= 'train_colsample_bytree:'   + str(value))\n",
    "\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'subsample' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5、调整叶子节点最小数据量和叶子节点最小权重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'min_child_samples': [20, 200, 2000], 'min_child_weight': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_child_samples=[20,200,2000]\n",
    "min_child_weight=[0.5,1,2]\n",
    "param_5=dict(min_child_samples=min_child_samples,min_child_weight=min_child_weight)\n",
    "param_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=0.8,\n",
       "        learning_rate=0.1, max_bin=256, max_depth=8, min_child_samples=20,\n",
       "        min_child_weight=1, min_split_gain=0.0, n_estimators=363,\n",
       "        n_jobs=-1, nthread=6, num_leaves=50, objective='binary',\n",
       "        random_state=None, reg_alpha=0, reg_lambda=0, scale_pos_weight=1,\n",
       "        seed=0, silent=True, subsample=0.8, subsample_for_bin=200000,\n",
       "        subsample_freq=1),\n",
       "       fit_params=None, iid=True, n_jobs=6,\n",
       "       param_grid={'min_child_samples': [20, 200, 2000], 'min_child_weight': [0.5, 1, 2]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=3)\n",
    "\n",
    "params = {'boosting_type': 'gbdt', \n",
    "          'objective': 'binary', \n",
    "          'nthread': 6, \n",
    "          'silent': True,\n",
    "          'learning_rate': 0.1, \n",
    "          'num_leaves': 50, \n",
    "          'max_depth': 8,\n",
    "          'max_bin': 256, \n",
    "          'subsample_for_bin': 200000,#默认200000，用来构建直方图数据数量\n",
    "          'subsample': 0.8, #默认1\n",
    "          'subsample_freq': 1, \n",
    "          'colsample_bytree': 0.8, \n",
    "          'reg_alpha': 0.8,          \n",
    "          'reg_lambda': 0.25,  \n",
    "          'min_split_gain': 0.0, \n",
    "          'min_child_weight': 1,         \n",
    "          'min_child_samples': 20, \n",
    "          'scale_pos_weight': 1,\n",
    "         }\n",
    "lgbm_2=lgbm.sklearn.LGBMClassifier(n_estimators=363,seed=0,**params)\n",
    "gsearch2_2=GridSearchCV(lgbm_2,param_grid=param_5,scoring='neg_log_loss',n_jobs=6, cv=kfold)\n",
    "gsearch2_2.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.399807 using {'min_child_samples': 20, 'min_child_weight': 2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XuYFvV9/vH3vQdATiKnBoQIItGg4KoLxmMwSovYirapStR4arhQUGuaFNOm+alNGrSpRSMN8YDkpJBUjTTV0qqhGPHAUkEOHhBQWbGKVBSMwh4+vz9mdnn2/LAwuwvcr+t6LueZ+c7MZ1aevfc7M898FRGYmZntbQXtXYCZme2fHDBmZpYJB4yZmWXCAWNmZplwwJiZWSYcMGZmlgkHjJmZZcIBY2ZmmXDAmJlZJorau4D21Ldv3xgyZEh7l2Fmtk9ZtmzZ+xHRr6V2B3TADBkyhLKysvYuw8xsnyLpzXza+RSZmZllwgFjZmaZcMCYmVkmHDBmZpYJB4yZmWXCAWNmZplwwJiZWSYO6O/BtNatL9zKK//3SnuXYWbWakf1PorpY6Znug/3YMzMLBPuwbRC1qlvZrY/cA/GzMwy4YAxM7NMOGDMzCwTDhgzM8uEA8bMzDLhgDEzs0w4YMzMLBMOGDMzy4QDxszMMuGAMTOzTGQaMJLGS3pV0uuSbmxk+RRJKyUtl/Q7SSPS+Z0k3Z8uWyFpbM46k9L5L0n6D0l9c5Zdm+5vtaTbsjw2MzNrXmYBI6kQmAWcDYwAJtUESI4HImJkRJQAtwG3p/O/BhARI4FxwD9JKpBUBNwBnBERo4CXgGnp/s4AJgKjIuJo4AdZHZuZmbUsyx7MGOD1iFgfETuBeSQBUCsiPsp52w2IdHoE8GTa5j1gK1AKKH11kySgJ7ApXedqYEZE7MhZz8zM2kmWAXMosDHnfXk6rw5JUyWtI+nBXJfOXgFMlFQkaShwAjA4IipIgmQlSbCMAO5L1/kccJqk5yX9t6TRWRyUmZnlJ8uAUSPzosGMiFkRMQyYDnw7nT2HJJDKgJnAEqBSUjFJwBwHDCQ5RfatdJ0i4BDgC8A3gV+mvZy6RUmTJZVJKtu8efMeHJ6ZmTUny4ApBwbnvB/ErtNZjZkHnAcQEZURcUNElETERKAXsBYoSZevi4gAfgmcnLO/hyPxAlAN9K2/k4i4OyJKI6K0X79+e3aEZmbWpCwDZikwXNJQSZ2Ai4AFuQ0kDc95ew5JiCCpq6Ru6fQ4oDIi1gBvAyMk1STDOODldPrXwJfSdT4HdALez+LAzMysZZmNaBkRlZKmAQuBQmBORKyWdAtQFhELgGmSzgIqgA+Ay9LV+wMLJVWThMql6TY3SboZWCypAngTuDxdZw4wR9IqYCdwWdrLMTOzdqAD+XdwaWlplJWVtXcZZmb7FEnLIqK0pXb+Jr+ZmWXCAWNmZplwwJiZWSYcMGZmlgkHjJmZZcIBY2ZmmXDAmJlZJhwwZmaWCQeMmZllwgFjZmaZcMCYmVkmHDBmZpYJB4yZmWXCAWNmZplwwJiZWSYcMGZmlgkHjJmZZcIBY2ZmmXDAmJlZJhwwZmaWCQeMmZllwgFjZmaZyDRgJI2X9Kqk1yXd2MjyKZJWSlou6XeSRqTzO0m6P122QtLYnHUmpfNfkvQfkvrW2+Y3JEX9+WZm1rYyCxhJhcAs4GxgBDCpJkByPBARIyOiBLgNuD2d/zWAiBgJjAP+SVKBpCLgDuCMiBgFvARMy9nn4LT9W1kdl5mZ5SfLHswY4PWIWB8RO4F5wMTcBhHxUc7bbkCk0yOAJ9M27wFbgVJA6aubJAE9gU052/hn4K9ztmNmZu0ky4A5FNiY8748nVeHpKmS1pH0YK5LZ68AJkoqkjQUOAEYHBEVwNXASpJgGQHcl27nXODtiFiR0fGYmdluyDJg1Mi8Bj2LiJgVEcOA6cC309lzSAKpDJgJLAEqJRWTBMxxwECSU2TfktQV+FvgOy0WJU2WVCapbPPmzbt/VGZmlpcsA6YcGJzzfhB1T2fVNw84DyAiKiPihogoiYiJQC9gLVCSLl8XEQH8EjgZGAYMBVZIeiPd1/9I+kz9nUTE3RFRGhGl/fr129NjNDOzJmQZMEuB4ZKGSuoEXAQsyG0gaXjO23NIQgRJXSV1S6fHAZURsQZ4GxghqSYZxgEvR8TKiOgfEUMiYghJuB0fEf+b4fGZmVkzirLacERUSpoGLAQKgTkRsVrSLUBZRCwApkk6C6gAPgAuS1fvDyyUVE0SKpem29wk6WZgsaQK4E3g8qyOwczMWk/JmaYDU2lpaZSVlbV3GWZm+xRJyyKitKV2/ia/mZllwgFjZmaZcMCYmVkmHDBmZpYJB4yZmWXCAWNmZplwwJiZWSYcMGZmlgkHjJmZZcIBY2ZmmXDAmJlZJhwwZmaWCQeMmZllwgFjZmaZcMCYmVkmHDBmZpYJB4yZmWXCAWNmZplwwJiZWSYcMGZmlgkHjJmZZcIBY2ZmmSjKcuOSxgN3AIXAvRExo97yKcBUoArYDkyOiDWSOgE/BkqBauD6iFiUrjMJ+BsggE3AJRHxvqR/BP4E2AmsA66IiK1ZHp/ZnqioqKC8vJxPP/20vUsxa1SXLl0YNGgQxcXFrVpfEbGXS0o3LBUCrwHjgHJgKTApItbktOkZER+l0+cC10TEeElTgdKIuEJSf+BxYDRJj2sTMCINlduA30fETZL+EHgqIiol3QoQEdObq7G0tDTKysr29qGb5WXDhg306NGDPn36IKm9yzGrIyLYsmUL27ZtY+jQoXWWSVoWEaUtbSPLU2RjgNcjYn1E7ATmARNzG9SES6obSa8EYATwZNrmPWArSW9G6aubkk9kT5LAISL+MyIq0/WfAwZlcVBme8unn37qcLEOSxJ9+vTZox52lgFzKLAx5315Oq8OSVMlrQNuA65LZ68AJkoqkjQUOAEYHBEVwNXAStKeDHBfI/u+kqTXY9ahOVysI9vTf59ZBkxjlTU4HxcRsyJiGDAd+HY6ew5JIJUBM4ElQKWkYpKAOQ4YCLwEfKvOTqW/BSqBXzRalDRZUpmkss2bN7fmuMzMLA8tBoykYZI6p9NjJV0nqVce2y4HBue8H0R6OqsJ84DzACKiMiJuiIiSiJgI9ALWAiXp8nWRXDz6JXByTq2XAX8MXBxNXFyKiLsjojQiSvv165fHYZjtn7Zu3cq//Mu/7PZ6EyZMYOvW7O6fOfnkk1tulKe5c+cybdq0RpflHkf37t0bbXP55Zfzr//6r3utnhqzZ8/mpz/9abNtmqv9H/7hH5pcb8OGDZx44okMHz6cCy+8kJ07dzZo88Ybb3DQQQdRUlJCSUkJU6ZM2b0DyFM+PZiHgCpJR5CcjhoKPJDHekuB4ZKGpneFXQQsyG0gaXjO23NIQgRJXSV1S6fHAZXpzQFvAyMk1STDOODltN14kl7QuRHx+zzqMzugNRUwVVVVza732GOP0atXPn9jts6SJUsy23aurI+jOVOmTOGrX/1qq9dvLmCmT5/ODTfcwNq1aznkkEO4777GriLAsGHDWL58OcuXL2f27NmtrqU5+dymXJ3emXU+MDMifijpxZZWSteZBiwkuU15TkSslnQLUBYRC4Bpks4CKoAPgMvS1fsDCyVVk4TKpek2N0m6GVgsqQJ4E7g8XecuoDPwX+l5w+ciIptYNtvLbv631azZ9FHLDXfDiIE9+X9/cnSTy2+88UbWrVtHSUkJxcXFdO/enQEDBrB8+XLWrFnDeeedx8aNG/n000+5/vrrmTx5MgBDhgyhrKyM7du3c/bZZ3PqqaeyZMkSDj30UB599FEOOuigRvc3duxYjjvuOJYtW8bmzZv56U9/yve//31WrlzJhRdeyHe/+10g6U1s376dRYsWcdNNN9G3b19WrVrFCSecwM9//vMmrwssXbqU66+/no8//pjOnTvz5JNPArBp0ybGjx/PunXrOP/887ntttvqHEffvn1rtxERXHvttTz11FMMHTqU5u6yfeGFF5gxYwYPP/wwjz76KBdddBEffvgh1dXVjBgxgvXr17Nu3TqmTp3K5s2b6dq1K/fccw9HHXUUN910E927d+cb3/gGS5cu5aqrrqJbt26ceuqpPP7446xatarJ2m+88UY++eQTSkpKOProo/nFL3ZdDYgInnrqKR54IOkDXHbZZdx0001cffXVTR5HlvIJmIr0uyeXkXzPBCCvm6Ij4jHgsXrzvpMzfX0T670BHNnEstlAg7iNiCPyqcnMEjNmzGDVqlUsX76cRYsWcc4557Bq1araW1LnzJlD7969+eSTTxg9ejR/9md/Rp8+fepsY+3atTz44IPcc889XHDBBTz00ENccsklTe6zU6dOLF68mDvuuIOJEyeybNkyevfuzbBhw7jhhhsabP/FF19k9erVDBw4kFNOOYVnnnmGU089tcF2d+7cyYUXXsj8+fMZPXo0H330UW3QLV++nBdffJHOnTtz5JFHcu211zJ48OAG2wB45JFHePXVV1m5ciXvvvsuI0aM4Morr2y07fHHH8+LLyZ/az/99NMcc8wxLF26lMrKSk488UQAJk+ezOzZsxk+fDjPP/8811xzDU899VSd7VxxxRXcfffdnHzyydx44411ljVW+4wZM7jrrrtYvnx5bbsJEyZw77330qlTJ3r16kVRUfKrfdCgQbz99tuN1r9hwwaOO+44evbsyXe/+11OO+20RtvtiXwC5gpgCvC9iNiQ3tX1871eidkBrLmeRlsZM2ZMne873HnnnTzyyCMAbNy4kbVr1zYIgKFDh1JSUgLACSecwBtvvNHsPs4991wARo4cydFHH82AAQMAOPzww9m4cWOD7Y8ZM4ZBg5JvHJSUlPDGG280GjCvvvoqAwYMYPTo0QD07NmzdtmZZ57JwQcfDMCIESN48803mwyYxYsXM2nSJAoLCxk4cCBf+tKXmjyWoqIijjjiCF5++WVeeOEFvv71r7N48WKqqqo47bTT2L59O0uWLOHP//zPa9fZsWNHnW1s3bqVbdu21V53+spXvsJvfvOb3a79sceSv+Mbu3GpsR7fgAEDeOutt+jTpw/Lli3jvPPOY/Xq1XV+bntDiwGTXvu4Li30EKBH/W/km9m+r1u3brXTixYt4oknnuDZZ5+la9eujB07ttHvQ3Tu3Ll2urCwkE8++aTZfdS0LygoqLNuQUEBlZWVTbav2X5jbSA5NdTUqbN8t1Fjd27NPe2003j88ccpLi7mrLPO4vLLL6eqqoof/OAHVFdX06tXrzo9jcbqbs7u1t63b1+2bt1KZWUlRUVFlJeXM3DgwEa3W7PtE044gWHDhvHaa69RWtridyd3Sz53kS2S1FNSb5Lvp9wv6fa9WoWZtbkePXqwbdu2Rpd9+OGHHHLIIXTt2pVXXnmF5557ro2r2z1HHXUUmzZtYunSpQBs27atxV/GjTn99NOZN28eVVVVvPPOO/z2t79tsf3MmTM56aST6NevH1u2bOGVV17h6KOPpmfPngwdOpRf/epXQBImK1asqLP+IYccQo8ePWp/vvPmzcurzuLiYioqKhrMl8QZZ5xRe+fbT37yEyZOnNig3ebNm2tv5li/fj1r167l8MMPz2vfuyOfu8gOTr9x/6fA/RFxAnDWXq/EzNpUnz59OOWUUzjmmGP45je/WWfZ+PHjqaysZNSoUfzd3/0dX/jCF9qpyvx06tSJ+fPnc+2113Lssccybty4Vn0D/fzzz2f48OGMHDmSq6++mi9+8YvNtj/xxBN59913Of300wEYNWoUo0aNqu0F/eIXv+C+++7j2GOP5eijj+bRRx9tsI377ruPyZMnc9JJJxERtafEmjN58mRGjRrFxRdfDCTXYDZtSr4Fcuutt3L77bdzxBFHsGXLFq666ioAFixYwHe+k1wCX7x4MaNGjeLYY4/ly1/+MrNnz6Z37955/pTy1+KzyCStBP4Q+AnwtxGxVNJLETFqr1fTxvwsMmtPL7/8Mp///OfbuwxrZ9u3b6/9Hs6MGTN45513uOOOO9q5ql0a+3ea77PI8rnIfwvJrcbPpOFyOOn3VczMbM/8+7//O9///veprKzksMMOY+7cue1d0l6Tz0X+XwG/ynm/HvizLIsys33X1KlTeeaZZ+rMu/7667niiiv2yvbPP/98NmzYUGferbfeyh/90R/tle239T4vvPBCLrzwwj3eTkfUYsBIGgT8EDiF5FlivyMZn6U849rMbB80a9asTLdfc+t0W2qPfe4P8rnIfz/JI14GkjwN+d/SeWZmZk3KJ2D6RcT96QMoKyNiLuCnRJqZWbPyCZj3JV0iqTB9XQJsybowMzPbt+UTMFcCFwD/C7wDfJnk8TFmZmZNajFgIuKtiDg3IvpFRP+IOI/kS5dmtg/zeDD753gwd911F0cccQSSeP/99/eozj3V2hEtv75XqzCzNufxYPbP8WBOOeUUnnjiCQ477LBWb39vyeeLlo3xQOJme9PjN8L/rty72/zMSDi76efSejyY/W88GIDjjjuuyZrbWmt7MM0/X8bMOrwZM2bUjmr4j//4j7zwwgt873vfY82aNUAyHsyyZcsoKyvjzjvvZMuWhvf2rF27lqlTp7J69Wp69erFQw891Ow+a8aDmTJlChMnTmTWrFmsWrWKuXPnNrr9F198kZkzZ7JmzRrWr1/f4AucNWrGg7njjjtYsWIFTzzxRJ3xYObPn8/KlSuZP38+GzdubLK+3PFg7rnnnmZ7U02NB/P888/XGQ/mhz/8IcuWLeMHP/gB11xzTYPtXHHFFcyePZtnn32WwsLCOssaq33GjBkcdNBBLF++vDZccp9F1pE02YORtI3Gg0RA43+imFnrNNPTaCseD2bfHw+mo2kyYCKiR1sWYmbty+PBJPbl8WA6mtaeIjOzfZzHg2loXx8PpqNxwJgdoDweTEP7w3gwd955J4MGDaK8vJxRo0bxF3/xF7v1M9ibWhwPZn/m8WCsPXk8GAOPB2NmZhk5oMeDaeJusg+BMuCv0vFhmlp3PHAHUAjcGxEz6i2fAkwFqoDtwOSIWCOpE/BjoBSoJhkeYFG6ziTgb9KaNgGXRMT7knoD84EhwBvABRHxQUvHZ2Z7l8eD2T3783gw+QyZfDPJL/IHSG5Rvgj4DPAqcHVEjG1ivULgNWAcUA4sBSZFxJqcNj0j4qN0+lzgmogYL2kqUBoRV0jqDzwOjCa5ZrQJGJGGym3A7yPipnT6/yJihqQbgUMiYnpzx+ZTZNaefIrM9gV7coosn4v84yPixxGxLSI+ioi7gQkRMR84pJn1xgCvR8T6iNgJzAMm5jaoCZdUN3b1lEYAT6Zt3gO2kvRmlL66KbmK1pMkcEi3/ZN0+ifAeXkcm5mZZSSfgKmWdIGkgvR1Qc6y5ro/hwK5X5ktT+fVIWmqpHXAbcB16ewVwERJRZKGAicAgyOiArgaWEnakwHuS9f5g4h4ByD9b/88js3MzDKST8BcDFwKvJe+LgUukXQQ0PijPhONfVupQSBFxKyIGAZMB76dzp5DEkhlwExgCVApqZgkYI4jGWHzJeBbeRzDrqKkyZLKJJVt3rx5d1Y1M7Pd0OJF/vQi/p80sfh3zaxaDuQ+02AQu05nNWYe8KN0n5XADTULJC0B1gIl6fJ16fxfAjemzd6VNCAi3pE0gCQMGzueu4G7IbkG00w9Zma2B1rswUgaJOkRSe9JelfSQ5IG5bHtpcBwSUPTu8IuAhbU2/bwnLfnkIQIkrpK6pZOjwMq05sD3gZGSKoZsnkc8HI6vQC4LJ2+DGj4jSYzq+XxYPbP8WAuvvhijjzySI455hiuvPLKdv3Gfz6nyO4n+eU9kOQayr+l85qV9kKmAQtJQuCXEbFa0i3pHWMA0yStlrScZIyZmoDoD/yPpJdJTp1dmm5zE3AzsFjSSyQ9mpqf9AxgnKS1JMHT/k8PNOvAPB7M/jkezMUXX8wrr7zCypUr+eSTT7j33ntbvZ89lc8XLftFRG6gzJX0l/lsPCIeAx6rN+87OdPXN7HeG8CRTSybDcxuZP4W4Mx86jLraG594VZe+b9X9uo2j+p9FNPHNH2nvseD2T/Hg5kwYULt9JgxYygvL2/yGLKWTw/mfUmXSCpMX5cADQduMLN9iseDaWh/Gg+moqKCn/3sZ4wfP77JY8haPj2YK4G7gH8muQtsCbB3vpJrZgDN9jTaiseD2b/Gg7nmmms4/fTTOe2005o8hqzlcxfZW8C5ufPSU2QzsyrKzNqex4NJ7A/jwdx8881s3ryZH//4x/kdSEZa+7j+r+/VKsyszXk8mIb2h/Fg7r33XhYuXMiDDz5IQUH7jsjS2r3nH/Fm1iF5PJiG9ofxYKZMmcK7777LSSedRElJCbfccstu/Qz2plaNByPprYj4bAb1tCk/7NLakx92aXCAjgfTxGP6Iem9NH4fopmZ7ZYDcjyYiOjRloWY2f7B48Hsnv15PBiPaGlme9WsWbMy3X7NrdNtqT32uT9o31sMzA5wrbkGatZW9vTfpwPGrJ106dKFLVu2OGSsQ4oItmzZQpcuXVq9DZ8iM2sngwYNory8HI9LZB1Vly5dap+k0BoOGLN2UlxcXOfRLGb7G58iMzOzTDhgzMwsEw4YMzPLhAPGzMwy4YAxM7NMOGDMzCwTDhgzM8uEA8bMzDLhgDEzs0xkGjCSxkt6VdLrkm5sZPkUSSslLZf0O0kj0vmdJN2fLlshaWw6v0fatub1vqSZ6bLPSvqtpBclvSRpQpbHZmZmzcvsUTGSCoFZwDigHFgqaUFErMlp9kBEzE7bnwvcDowHvgYQESMl9QcelzQ6IrYBJTn7WAY8nL79NvDLiPhRGlSPAUOyOj4zM2telj2YMcDrEbE+InYC84CJuQ0i4qOct93YNYLmCODJtM17wFagzvCckoYD/YGnazYH9EynDwY27bUjMTOz3ZZlwBwKbMx5X57Oq0PSVEnrgNuA69LZK4CJkookDQVOAAbXW3USMD92Pev8JuASSeUkvZdr99aBmJnZ7ssyYNTIvAYDX0TErIgYBkwnOc0FMIckkMqAmcASoLLeqhcBD+a8nwTMjYhBwATgZ5IaHJ+kyZLKJJX5MelmZtnJMmDKqdvrGETzp63mAecBRERlRNwQESURMRHoBaytaSjpWKAoIpblrH8V8Mt0/WeBLkDf+juJiLsjojQiSvv169e6IzMzsxZlGTBLgeGShkrqRNLjWJDbIL2OUuMc0hCR1FVSt3R6HFBZ7+aASdTtvQC8BZyZrvN5koBxF8XMrJ1kdhdZRFRKmgYsBAqBORGxWtItQFlELACmSToLqAA+AC5LV+8PLJRUDbwNXFpv8xeQnAbL9VfAPZJuIDkVd3l4LFozs3ajA/l3cGlpaZSVlbV3GWZm+xRJyyKitKV2/ia/mZllwgFjZmaZcMCYmVkmHDBmZpYJB4yZmWXCAWNmZplwwJiZWSYcMGZmlgkHjJmZZcIBY2ZmmXDAmJlZJhwwZmaWCQeMmZllIrPH9e/XXlsIqx5q7yrM9iGNDXBr7arkK3D4FzPdhQOmNT7aBBufb+8qzPYNB/CQIB3aEWdmvgsHTGuUXpG8zMysSb4GY2ZmmXDAmJlZJhwwZmaWCQeMmZllwgFjZmaZcMCYmVkmHDBmZpaJTANG0nhJr0p6XdKNjSyfImmlpOWSfidpRDq/k6T702UrJI1N5/dI29a83pc0M2d7F0haI2m1pAeyPDYzM2teZl+0lFQIzALGAeXAUkkLImJNTrMHImJ22v5c4HZgPPA1gIgYKak/8Lik0RGxDSjJ2ccy4OF0ejjwLeCUiPggXc/MzNpJlj2YMcDrEbE+InYC84CJuQ0i4qOct92AmmdKjACeTNu8B2wFSnPXTQOlP/B0OutrwKyI+CBnPTMzaydZBsyhwMac9+XpvDokTZW0DrgNuC6dvQKYKKlI0lDgBGBwvVUnAfMjah909Dngc5KekfScpPGNFSVpsqQySWWbN29u9cGZmVnzsgyYxh6f2uCpdxExKyKGAdOBb6ez55AEUhkwE1gCVNZb9SLgwZz3RcBwYCxJ+NwrqVcj+7s7IkojorRfv367dUBmZpa/LB92WU7dXscgYFMz7ecBPwKIiErghpoFkpYAa3PeHwsURcSyevt7LiIqgA2SXiUJnKV7eBxmZtYKWfZglgLDJQ2V1Imkx7Egt0F6HaXGOaQhIqmrpG7p9Digst7NAZOo23sB+DVwRrpOX5JTZuv33uGYmdnuyKwHExGVkqYBC4FCYE5ErJZ0C1AWEQuAaZLOAiqAD4DL0tX7AwslVQNvA5fW2/wFwIR68xYCfyhpDVAFfDMitmRxbGZm1jLFATwYUGlpaZSVlbV3GWZm+xRJyyKitKV2/ia/mZllwgFjZmaZcMCYmVkmHDBmZpYJB4yZmWXCAWNmZplwwJiZWSYcMGZmlgkHjJmZZcIBY2ZmmXDAmJlZJhwwZmaWCQeMmZllwgFjZmaZyHJES7M2UVUdfLyzko93JKNqFxUUUFQgigpFcWEyXVggpMZG8TazrDhgrF1UVwe/r6hi+6eVbN+RvD7eUcm2T5P/bs99pfO21UzvrKyz3u93VuW1z9rQKSigqFAUFRY0nFf732R5cc28tF1RYQHFBaKwIF2WLi8uzJmXbqPZeen2a/ZdWKB0e8m+igvrzqtpXzsv3a5D0zoyB4zlLSL4pF4o1ARAbUCk/03mVbF9R0Xarortn1bw8Y6qpO3OSvIZ6664UHTvXET3LkV061REjy5F9O7Wic/27prMT5d171xEt85FCKioDiqrqqmsCiqqq6mqil3zqoOKqmqqqoOKqibmVafrVlWzo6Ka7dVVdbdXHbXLc9etWd6WY/gVpr2z4nqB2Gxg1YZdI/MKCigsVJ0ALCrYFWi1oVw/gOuFaIN59QK2pqbcfebWafsHB8x+LiLYUVndsGdQLyTy6T18vKOS6jx+eRYWaNcv/zQADj6omEG9DqoNgiQUCuneuZhunQvp0aVo13T63+5diuhcVJj9D2kvq2okdGoCqao6CbAk3HYFVkUaYDXLmp2XhmXtvHT7lVXVVFRHGqjNb++/n0XRAAAL/ElEQVTTyrohWVn739z60nnVyfptRaK2V5d7mrMmfBrtheYEVmOnSOv0VhubV6/nWnd7dUO5tjdbf15tTY33iA/E3qYDpoPaUVmV/nKvYtuOmr/8K9IQqOkZVNUGwPYGvYddr3x+ORQIunUuokedACjiMz271IZCjy7psprpTrva5bbpXFRwQH6YaiS9in0vGJtTnQZNZXVOEKWBlhtOjQVWbgBW1gvYynpBWDsvJ0R3tam3z6qos73f76ysE5J19l2vV1szry3VD6aieqdZ6/YKcwOr5hRpvus2PM2b9HLrbu/ogQfz2T5dsz3mTLd+gKmoqm54TSGnF9Bi7yHn2kJFVcv/+CWSX/Kdi9K/+Ivp0bmIvt070b1zcdJDSEOhR85pptxQqOlhHFRceECHgjWvoEB0KhCd9qMbTyMi7VHm9u7qniJtrlfXUgDmnoZtMC9ne0kvtO72anqcOyur+XhnVc66dcO8wbzdOEX73fOO4ZI+h2X6M3bAtMKCFZv4+XNv1rngvG1HJTsrq/Nav2unwjrXDrp3LmJw764Neg/1ewZ1eg+di+haXEiBz1ebtYpUc+MGdCnef3qc1XVOmzZ9GvYPenbJvBYHTCtEBAWCgb261LmmUBsQ9S4+116k7pz0IHwR08yyUlAgOhcU0rkD/HbPtARJ44E7gELg3oiYUW/5FGAqUAVsByZHxBpJnYAfA6VANXB9RCyS1AN4OmcTg4CfR8Rf5mzzy8CvgNERUZbFcU0sOZSJJYdmsWkzs/1GZgEjqRCYBYwDyoGlkhZExJqcZg9ExOy0/bnA7cB44GsAETFSUn/gcUmjI2IbUJKzj2XAwznvewDXAc9ndVxmZpafLK/YjQFej4j1EbETmAdMzG0QER/lvO0G1FyeGgE8mbZ5D9hK0pupJWk40J+6PZq/B24DPt17h2FmZq2RZcAcCmzMeV+ezqtD0lRJ60iC4bp09gpgoqQiSUOBE4DB9VadBMyPSO6ZkHQcMDgiftNcUZImSyqTVLZ58+bWHJeZmeUhy4Bp7Ep2gxvoImJWRAwDpgPfTmfPIQmkMmAmsASorLfqRcCDAJIKgH8G/qqloiLi7ogojYjSfv365XkoZma2u7K8yF9O3V7HIGBTM+3nAT8CiIhK4IaaBZKWAGtz3h8LFEXEsnRWD+AYYFH6XY7PAAsknZvVhX4zM2telj2YpcBwSUPTu8IuAhbkNkivo9Q4hzREJHWV1C2dHgdU1rs5YBJp7wUgIj6MiL4RMSQihgDPAQ4XM7N2lFkPJiIqJU0DFpLcpjwnIlZLugUoi4gFwDRJZwEVwAfAZenq/YGFkqqBt4FL623+AmBCVrWbmdmeU7Tlo187mNLS0igrcyfHzGx3SFoWEaUttjuQA0bSZuDN9q4jR1/g/fYuohkdvT7o+DV29Pqg49fY0euD/b/GwyKixbukDuiA6WgkleXzV0F76ej1QcevsaPXBx2/xo5eH7jGGvvPo1HNzKxDccCYmVkmHDAdy93tXUALOnp90PFr7Oj1QcevsaPXB64R8DUYMzPLiHswZmaWCQdMG5M0XtKrkl6XdGMTbS6QtEbSakkPdLQaJX1W0m8lvSjpJUlt+qVXSXMkvSdpVRPLJenOtP6XJB3fweq7OK3rJUlL0kcftamWasxpN1pSVTrOUpvKp0ZJYyUtTz8r/92R6pN0sKR/k7Qire+KNq5vcPo5fTnd//WNtMn2sxIRfrXRi+SJBuuAw4FOJE+NHlGvzXDgReCQ9H3/Dljj3cDV6fQI4I02rvF04HhgVRPLJwCPkzxw9QvA8x2svpNz/v+e3db15VNjzr+Fp4DHgC93tBqBXsAa4LPp+7b+rLRU398At6bT/YD/Azq1YX0DgOPT6R7Aa418ljP9rLgH07ZaHCOHZLC1WRHxAdSOh9PRagygZzp9MM0/xHSvi4jFJB/WpkwEfhqJ54Bekga0TXUt1xcRS2r+/5I8N29QmxRWt4aWfoYA1wIPAW39bxDIq8avAA9HxFtp+zatM4/6Auih5Am83dO29Z8Kn5mIeCci/ied3ga8TMMhUzL9rDhg2lY+Y+R8DvicpGckPZcOO92W8qnxJuASSeUkf91e2zal5S2vsYg6iKtI/oLsUCQdCpwPzG7vWprxOeAQSYskLZP01fYuqJ67gM+T/AG2kmTo9+r2KETSEOA4Go72m+lnJcvH9VtD+YyRU0RymmwsyV+2T0s6JiK2ZlxbjXxqnATMjYh/knQS8LO0xnb58DQir7GI2pukM0gC5tT2rqURM4HpEVGVDoHRERWRDEZ4JnAQ8Kyk5yLitfYtq9YfAcuBLwHDgP+S9HTUHck3c5K6k/RE/7KRfWf6WXHAtK18xsgpB56LiApgg6RXSQJnaduUmFeNVwHjASLiWUldSJ5r1C6nUhqxu2MRtTlJo4B7gbMjYkt719OIUmBeGi59gQmSKiPi1+1bVh3lwPsR8THwsaTFwLEk1xo6giuAGZFc7Hhd0gbgKOCFtipAUjFJuPwiIh5upEmmnxWfImtbLY6RA/waOANAUl+S0wDrO1iNb5H81YikzwNdgI40/vQC4KvpHTJfAD6MiHfau6gakj4LPAxc2oH+2q4jIobGrvGV/hW4poOFC8CjwGlKhlbvCpxIcp2ho8j9nPwBcCRt+FlOr/3cB7wcEbc30SzTz4p7MG0o8hsjZyHwh5LWAFXAN9vyL9w8a/wr4B5JN5B0py9P/0prE5IeJDmF2De9DvT/gOK0/tkk14UmAK8Dvyf5S7LN5FHfd4A+wL+kPYTKaOMHI+ZRY7trqcaIeFnSfwAvAdXAvRHR7G3XbVkf8PfAXEkrSU5FTY+ItnzC8ikkY2mtlLQ8nfc3wGdzasz0s+Jv8puZWSZ8iszMzDLhgDEzs0w4YMzMLBMOGDMzy4QDxszMMuGAMTOzTDhg7IAn6Vw1MXRCK7e3SFKD77VIKpV0Zzp9uaS7mlh/+96qZU9JeiP9wq/ZbvMXLe2Al355tP7TCrLYTxlQlvV+zDoK92BsvyZpiKRXJN0raZWkX0g6K31a9VpJY3J7E5LmpgMwLZG0vqWBtiT9taSV6aBSM3IW/bmkFyS9Jum0tO1YSb9pZBtDJT0raamkv29hfwMkLVYyyNaqnG3/SFKZkoGlbs5p/4akf0i3XybpeEkLJa2TNCWnrsWSHlEy0N1sSQ1+N0i6JD2m5ZJ+LKkwfc1Na1mZPt3BDHDA2IHhCOAOYBTJwwa/QvIE42+QPDqjvgHp8j8GZjSyHABJZwPnASdGxLHAbTmLiyJiDPCXJI8Qac4dwI8iYjTwvy20/QqwMCJKSB7sWPMIkL9NHzczCvhi+jDNGhsj4iTgaWAu8GWSwaVuyWkzhuQRQCNJnvz7p/WO9fPAhcAp6b6rgIuBEuDQiDgmIkYC97dQvx1AHDB2INgQESvT4QRWA0+mz05bCQxppP2vI6I6ItYAf9DMds8C7o+I3wNERO7gUzVPrl3WxD5ynQI8mE7/rIW2S4ErJN0EjEwHkgK4QNL/kIyGejTJSKM1ak7/rSQZsXBbRGwGPpXUK132QjrIXFVaS/0hBM4keTT+0vS5VmeSjHq6Hjhc0g+VjF3Upo+it47NAWMHgh0509U576tp/DpkbvvmBkMRTY+dUbONqib2UV9eDwVMR1E8HXibZByer0oaStIbOzMiRgH/TvKE6/q15B57zfua2urvv/57AT+JiJL0dWRE3JSOzHkssAiYSjIEgRnggDHbE/8JXJk+Kh5JvVu5nWdIhkWA5LRTkyQdBrwXEfeQPIr9eJLhqz8GPkwfC392K2oYk14LKiA5Ffa7esufBL4sqX9aR29Jh6V3mBVExEPA36X1mAG+i8ys1SLiPySVAGWSdpI8+ryxazotuR54QNL1JINDNWcs8E1JFcB24KsRsUHSiySn/9aTBNbuepbketNIYDHwSO7CiFgj6dvAf6YhVEHSY/kEuD/npoBvtWLftp/y4/rNDnCSxgLfiIg/bu9abP/iU2RmZpYJ92DMWiBpJA3v7toRESfuT/s029scMGZmlgmfIjMzs0w4YMzMLBMOGDMzy4QDxszMMuGAMTOzTPx/T5eis+u+D5cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2a68eeeb470>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_2.best_score_, gsearch2_2.best_params_))\n",
    "test_means = gsearch2_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "pd.DataFrame(gsearch2_2.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(subsample), len(colsample_bytree))\n",
    "train_scores = np.array(train_means).reshape(len(subsample), len(colsample_bytree))\n",
    "\n",
    "for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(min_child_samples, -test_scores[i], label= 'valid_min_child_weight:'   + str(value))\n",
    "     pyplot.plot(min_child_weight, -train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'min_child_samples' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6 重新优化一下 树的数量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'boosting_type': 'gbdt', \n",
    "          'objective': 'binary', \n",
    "          'nthread': 6, \n",
    "          #default=OpenMP_default, type=int, alias=num_thread, nthread LightGBM 的线程数\n",
    "          'silent': True,\n",
    "          'learning_rate': 0.1, \n",
    "          'num_leaves': 50, \n",
    "          #num_leaves：叶子数\n",
    "#num_leaves, default=31, type=int, alias=num_leaf一棵树上的叶子数\n",
    "          'max_depth': 8,\n",
    "          'max_bin': 256, \n",
    "          'subsample_for_bin': 200000,#默认200000，用来构建直方图数据数量\n",
    "          'subsample': 0.8, #默认1\n",
    "          #类似于feature_fraction,它将在不进行重采样的情况下\n",
    "          #随机选择部分数据可以用来加速训练可以用来处理过拟合Not\n",
    "          'subsample_freq': 1, \n",
    "          'colsample_bytree': 0.8,       \n",
    "          'reg_alpha': 0.8,         \n",
    "          'reg_lambda': 0.25, \n",
    "          'min_split_gain': 0.0, \n",
    "          'min_child_weight': 1,        \n",
    "          'min_child_samples': 100, \n",
    "          #default=20, type=int, alias=min_data_per_leaf , min_data, \n",
    "          #min_child_samples一个叶子上数据的最小数量. 可以用来处理过拟合.\n",
    "        \n",
    "\n",
    "         }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1040: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n",
      "C:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:661: UserWarning: silent keyword has been found in `params` and will be ignored. Please use silent argument of the Dataset constructor to pass this parameter.\n",
      "  'Please use {0} argument of the Dataset constructor to pass this parameter.'.format(key))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best n_estimators: 284\n",
      "best cv logloss: 0.39955036588252835\n",
      "logloss of train : 0.39974860472393803\n",
      "logloss of test : 0.4045745986483505\n",
      "time cost232.7710998058319s\n"
     ]
    }
   ],
   "source": [
    "time_start=time.time()\n",
    "\n",
    "lgbm6 = lgbm.sklearn.LGBMClassifier(n_estimators=1000,seed=0, **params)\n",
    "modelfit(params,lgbm6, X_train, y_train)\n",
    "\n",
    "time_end=time.time()\n",
    "a=time_end - time_start\n",
    "print('time cost'+str(a)+'s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\lightgbm\\engine.py:102: UserWarning: Found `num_boost_round` in params. Will use it instead of argument\n",
      "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "92.0268828868866\n"
     ]
    }
   ],
   "source": [
    "start=time.time()\n",
    "# 定义模型\n",
    "lgb7 = lgbm.LGBMRegressor(\n",
    "          num_boost_round=284,\n",
    "          boosting='gbdt', \n",
    "          objective='binary', \n",
    "          num_threads=6, \n",
    "          #default=OpenMP_default, type=int, alias=num_thread, nthread LightGBM 的线程数\n",
    "#          'silent': True,\n",
    "          learning_rate=0.1, \n",
    "          num_leaves=50, \n",
    "          #num_leaves：叶子数\n",
    "#num_leaves, default=31, type=int, alias=num_leaf一棵树上的叶子数\n",
    "          max_depth= 8,\n",
    "          max_bi= 256, \n",
    "          subsample_for_bin= 200000,#默认200000，用来构建直方图数据数量\n",
    "          subsample= 0.8, #默认1\n",
    "          #类似于feature_fraction,它将在不进行重采样的情况下\n",
    "          #随机选择部分数据可以用来加速训练可以用来处理过拟合Not\n",
    "          subsample_freq= 1, \n",
    "          colsample_bytree= 0.8,       \n",
    "          reg_alpha= 0.8,         \n",
    "          reg_lambda= 0.25, \n",
    "          min_split_gain=0.0, \n",
    "          min_child_weight= 1,        \n",
    "          min_child_samples= 200, \n",
    "          min_data_in_leaf=50, \n",
    "          #default=20, type=int, alias=min_data_per_leaf , min_data, \n",
    "          #min_child_samples一个叶子上数据的最小数量. 可以用来处理过拟合.\n",
    "        \n",
    "        # 'categorical_feature':3\n",
    "         # 'categorical_feature':3:26\n",
    "         )\n",
    "# 训练学习\n",
    "model=lgb7.fit(X_train, y_train)\n",
    "end=time.time()\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of train : 0.4001760588598642\n",
      "logloss of valid : 0.40093575026031686\n",
      "logloss of test : 0.40354506207393454\n"
     ]
    }
   ],
   "source": [
    "# train上logloss评测\n",
    "y_pred_train = lgb7.predict(X_train)\n",
    "logloss = log_loss(y_train, y_pred_train)\n",
    "print (\"logloss of train :\",logloss )\n",
    "\n",
    "# valid上logloss评测\n",
    "y_pred_valid = lgb7.predict(X_valid)\n",
    "logloss = log_loss(y_valid, y_pred_valid)\n",
    "print (\"logloss of valid :\",logloss )\n",
    "\n",
    "# test上logloss评测\n",
    "y_pred_test = lgb7.predict(X_test)\n",
    "logloss = log_loss(y_test, y_pred_test)\n",
    "print (\"logloss of test :\",logloss )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
